
"""simple docstring"""





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
				)
				_a														=		None
_a														=		{
    """7B""": 1_1008,
    """13B""": 1_3824,
    """30B""": 1_7920,
    """65B""": 2_2016,
    """70B""": 2_8672,
}
_a														=		{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}



def       lowerCamelCase__		(			__snake_case,     __snake_case=1,     __snake_case=2_56	)							->		List[str]:


					"""simple docstring"""





					return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)



def       lowerCamelCase__		(			__snake_case	)							->		Dict:


					"""simple docstring"""





					with open(__UpperCAmelCase,     '''r'''	) as f:
										return json.load(__UpperCAmelCase	)



def       lowerCamelCase__		(			__snake_case,     __snake_case	)							->		Optional[int]:


					"""simple docstring"""





					with open(__UpperCAmelCase,     '''w'''	) as f:
										json.dump(__UpperCAmelCase,     __UpperCAmelCase	)



def       lowerCamelCase__		(			__snake_case,     __snake_case,     __snake_case,     __snake_case=True	)							->		int:


					"""simple docstring"""





					os.makedirs(__UpperCAmelCase,     exist_ok=__UpperCAmelCase	)
					_UpperCamelCase							 =  os.path.join(__UpperCAmelCase,     '''tmp'''	)
					os.makedirs(__UpperCAmelCase,     exist_ok=__UpperCAmelCase	)

					_UpperCamelCase							 =  read_json(os.path.join(__UpperCAmelCase,     '''params.json'''	)	)
					_UpperCamelCase							 =  NUM_SHARDS[model_size]
					_UpperCamelCase							 =  params['''n_layers''']
					_UpperCamelCase							 =  params['''n_heads''']
					_UpperCamelCase							 =  n_heads // num_shards
					_UpperCamelCase							 =  params['''dim''']
					_UpperCamelCase							 =  dim // n_heads
					_UpperCamelCase							 =  10000.0
					_UpperCamelCase							 =  1.0 / (base ** (torch.arange(0,     __UpperCAmelCase,     2	).float() / dims_per_head))

					if "n_kv_heads" in params:
										_UpperCamelCase							 =  params['''n_kv_heads''']  # for GQA / MQA
										_UpperCamelCase							 =  n_heads_per_shard // num_key_value_heads
										_UpperCamelCase							 =  dim // num_key_value_heads
					else:  # compatibility with other checkpoints
										_UpperCamelCase							 =  n_heads
										_UpperCamelCase							 =  n_heads_per_shard
										_UpperCamelCase							 =  dim

					# permute for sliced rotary
					def permute(__snake_case,     __snake_case=n_heads,     __snake_case=dim,     __snake_case=dim	):
										return w.view(__UpperCAmelCase,     dima // n_heads // 2,     2,     __UpperCAmelCase	).transpose(1,     2	).reshape(__UpperCAmelCase,     __UpperCAmelCase	)

					print(F'''Fetching all parameters from the checkpoint at {input_base_path}.'''	)
					# Load weights
					if model_size == "7B":
										# Not sharded
										# (The sharded implementation would also work, but this is simpler.)
										_UpperCamelCase							 =  torch.load(os.path.join(__UpperCAmelCase,     '''consolidated.00.pth'''	),     map_location='''cpu'''	)
					else:
										# Sharded
										_UpperCamelCase							 =  [
										    torch.load(os.path.join(__UpperCAmelCase,     F'''consolidated.{i:02d}.pth'''	),     map_location='''cpu'''	)
										    for i in range(__UpperCAmelCase	)
										]
					_UpperCamelCase							 =  0
					_UpperCamelCase							 =  {'''weight_map''': {}}
					for layer_i in range(__UpperCAmelCase	):
										_UpperCamelCase							 =  F'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
										if model_size == "7B":
															# Unsharded
															_UpperCamelCase							 =  {
															    F'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
															        loaded[F'''layers.{layer_i}.attention.wq.weight''']	),
															    F'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
															        loaded[F'''layers.{layer_i}.attention.wk.weight''']	),
															    F'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[F'''layers.{layer_i}.attention.wv.weight'''],
															    F'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[F'''layers.{layer_i}.attention.wo.weight'''],
															    F'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w1.weight'''],
															    F'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w2.weight'''],
															    F'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w3.weight'''],
															    F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[F'''layers.{layer_i}.attention_norm.weight'''],
															    F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[F'''layers.{layer_i}.ffn_norm.weight'''],
															}
										else:
															# Sharded
															# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
															# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
															# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

															_UpperCamelCase							 =  {
															    F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
															        F'''layers.{layer_i}.attention_norm.weight'''
															    ].clone(),
															    F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
															        F'''layers.{layer_i}.ffn_norm.weight'''
															    ].clone(),
															}
															_UpperCamelCase							 =  permute(
															    torch.cat(
															        [
															            loaded[i][F'''layers.{layer_i}.attention.wq.weight'''].view(__UpperCAmelCase,     __UpperCAmelCase,     __UpperCAmelCase	)
															            for i in range(__UpperCAmelCase	)
															        ],     dim=0,     ).reshape(__UpperCAmelCase,     __UpperCAmelCase	)	)
															_UpperCamelCase							 =  permute(
															    torch.cat(
															        [
															            loaded[i][F'''layers.{layer_i}.attention.wk.weight'''].view(
															                __UpperCAmelCase,     __UpperCAmelCase,     __UpperCAmelCase	)
															            for i in range(__UpperCAmelCase	)
															        ],     dim=0,     ).reshape(__UpperCAmelCase,     __UpperCAmelCase	),     __UpperCAmelCase,     __UpperCAmelCase,     __UpperCAmelCase,     )
															_UpperCamelCase							 =  torch.cat(
															    [
															        loaded[i][F'''layers.{layer_i}.attention.wv.weight'''].view(
															            __UpperCAmelCase,     __UpperCAmelCase,     __UpperCAmelCase	)
															        for i in range(__UpperCAmelCase	)
															    ],     dim=0,     ).reshape(__UpperCAmelCase,     __UpperCAmelCase	)

															_UpperCamelCase							 =  torch.cat(
															    [loaded[i][F'''layers.{layer_i}.attention.wo.weight'''] for i in range(__UpperCAmelCase	)],     dim=1	)
															_UpperCamelCase							 =  torch.cat(
															    [loaded[i][F'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(__UpperCAmelCase	)],     dim=0	)
															_UpperCamelCase							 =  torch.cat(
															    [loaded[i][F'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(__UpperCAmelCase	)],     dim=1	)
															_UpperCamelCase							 =  torch.cat(
															    [loaded[i][F'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(__UpperCAmelCase	)],     dim=0	)

										_UpperCamelCase							 =  inv_freq
										for k, v in state_dict.items():
															_UpperCamelCase							 =  filename
															param_count += v.numel()
										torch.save(__UpperCAmelCase,     os.path.join(__UpperCAmelCase,     __UpperCAmelCase	)	)

					_UpperCamelCase							 =  F'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
					if model_size == "7B":
										# Unsharded
										_UpperCamelCase							 =  {
										    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
										    '''model.norm.weight''': loaded['''norm.weight'''],
										    '''lm_head.weight''': loaded['''output.weight'''],
										}
					else:
										_UpperCamelCase							 =  {
										    '''model.norm.weight''': loaded[0]['''norm.weight'''],
										    '''model.embed_tokens.weight''': torch.cat(
										        [loaded[i]['''tok_embeddings.weight'''] for i in range(__UpperCAmelCase	)],     dim=1	),
										    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__UpperCAmelCase	)],     dim=0	),
										}

					for k, v in state_dict.items():
										_UpperCamelCase							 =  filename
										param_count += v.numel()
					torch.save(__UpperCAmelCase,     os.path.join(__UpperCAmelCase,     __UpperCAmelCase	)	)

					# Write configs
					_UpperCamelCase							 =  {'''total_size''': param_count * 2}
					write_json(__UpperCAmelCase,     os.path.join(__UpperCAmelCase,     '''pytorch_model.bin.index.json'''	)	)
					_UpperCamelCase							 =  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
					_UpperCamelCase							 =  params['''multiple_of'''] if '''multiple_of''' in params else 2_56
					_UpperCamelCase							 =  LlamaConfig(
					    hidden_size=__UpperCAmelCase,     intermediate_size=compute_intermediate_size(__UpperCAmelCase,     __UpperCAmelCase,     __UpperCAmelCase	),     num_attention_heads=params['''n_heads'''],     num_hidden_layers=params['''n_layers'''],     rms_norm_eps=params['''norm_eps'''],     num_key_value_heads=__UpperCAmelCase,     )
					config.save_pretrained(__UpperCAmelCase	)

					# Make space so we can load the model properly now.
					del state_dict
					del loaded
					gc.collect()

					print('''Loading the checkpoint in a Llama model.'''	)
					_UpperCamelCase							 =  LlamaForCausalLM.from_pretrained(__UpperCAmelCase,     torch_dtype=torch.floataa,     low_cpu_mem_usage=__UpperCAmelCase	)
					# Avoid saving this as part of the config.
					del model.config._name_or_path

					print('''Saving in the Transformers format.'''	)
					model.save_pretrained(__UpperCAmelCase,     safe_serialization=__UpperCAmelCase	)
					shutil.rmtree(__UpperCAmelCase	)



def       lowerCamelCase__		(			__snake_case,     __snake_case	)							->		List[Any]:


					"""simple docstring"""





					_UpperCamelCase							 =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
					print(F'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''	)
					_UpperCamelCase							 =  tokenizer_class(__UpperCAmelCase	)
					tokenizer.save_pretrained(__UpperCAmelCase	)



def       lowerCamelCase__		(			)							->		int:


					"""simple docstring"""





					_UpperCamelCase							 =  argparse.ArgumentParser()
					parser.add_argument(
					    '''--input_dir''',     help='''Location of LLaMA weights, which contains tokenizer.model and model folders''',     )
					parser.add_argument(
					    '''--model_size''',     choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''],     )
					parser.add_argument(
					    '''--output_dir''',     help='''Location to write HF model and tokenizer''',     )
					parser.add_argument('''--safe_serialization''',     type=__UpperCAmelCase,     help='''Whether or not to save using `safetensors`.'''	)
					_UpperCamelCase							 =  parser.parse_args()
					if args.model_size != "tokenizer_only":
										write_model(
										    model_path=args.output_dir,     input_base_path=os.path.join(args.input_dir,     args.model_size	),     model_size=args.model_size,     safe_serialization=args.safe_serialization,     )
					_UpperCamelCase							 =  os.path.join(args.input_dir,     '''tokenizer.model'''	)
					write_tokenizer(args.output_dir,     __UpperCAmelCase	)


if __name__ == "__main__":
				main()













from __future__ import annotations




def 		lowerCAmelCase_    (  __UpperCAmelCase:			list[int]		,   __UpperCAmelCase:			int      )      ->	list[int]:
    UpperCamelCase__							:				Optional[int]          =      0
    UpperCamelCase__							:				Tuple          =      len(__UpperCAmelCase      ) - 1

    while i < j:
        if nums[i] + nums[j] == target:
            return [i, j]
        elif nums[i] + nums[j] < target:
            UpperCamelCase__							:				Tuple          =      i + 1
        else:
            UpperCamelCase__							:				str          =      j - 1

    return []


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  print(F'''{two_pointer([2, 7, 11, 15], 9) = }''')

