

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_lowerCamelCase				:   Any					   =							[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_lowerCamelCase				:   List[str]					   =							[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]

def 			a__     (		UpperCAmelCase							:			Union[str, Any]				, UpperCAmelCase							:			List[str]			)  ->						Optional[Any]:
  UpperCAmelCase       :							Any  = {
      '''word_embeddings.weight''': '''word_embeddings.weight''',
      '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
      '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
      '''weight''': '''ln_f.weight''',
      '''bias''': '''ln_f.bias''',
  }

  if key in layer_rename_map:
    return layer_rename_map[key]

  # Handle transformer blocks
  UpperCAmelCase       :							int  = int(re.match(r'''.*layer_(\d*).*'''				, __snake_case			)[1]			)
  layer_number -= 3
  return f'''h.{layer_number}.''' + key

def 			a__     (		UpperCAmelCase							:			List[Any]			)  ->						List[Any]:





  if dtype == torch.bool:
    return 1 / 8
  UpperCAmelCase       :							Optional[Any]  = re.search(r'''[^\d](\d+)$'''				, str(__snake_case			)			)
  if bit_search is None:
    raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''			)
  UpperCAmelCase       :							Union[str, Any]  = int(bit_search.groups()[0]			)
  return bit_size // 8

def 			a__     (		UpperCAmelCase							:			Optional[Any]				, UpperCAmelCase							:			int				, UpperCAmelCase							:			List[Any]				, UpperCAmelCase							:			Dict				, UpperCAmelCase							:			Tuple			)  ->						Tuple:





  if bloom_config_file == "":
    UpperCAmelCase       :							Optional[Any]  = BloomConfig()
  else:
    UpperCAmelCase       :							Union[str, Any]  = BloomConfig.from_json_file(__snake_case			)

  if shard_model:
    UpperCAmelCase       :							Union[str, Any]  = os.listdir(__snake_case			)
    UpperCAmelCase       :							Union[str, Any]  = sorted(filter(lambda UpperCAmelCase			: s.startswith('''layer'''			) and "model_00" in s				, __snake_case			)			)

    UpperCAmelCase       :							Union[str, Any]  = {'''weight_map''': {}, '''metadata''': {}}
    UpperCAmelCase       :							Tuple  = 0

    UpperCAmelCase       :							Union[str, Any]  = None

    UpperCAmelCase       :							List[Any]  = BloomConfig()

    for j, file in enumerate(__snake_case			):
      print('''Processing file: {}'''.format(__snake_case			)			)
      UpperCAmelCase       :							Any  = None

      for i in range(__snake_case			):
        # load all TP files
        UpperCAmelCase       :							str  = file.replace('''model_00'''				, f'''model_0{i}'''			)
        UpperCAmelCase       :							Tuple  = torch.load(os.path.join(__snake_case				, __snake_case			)				, map_location='''cpu'''			)

        # Rename keys in the transformers names
        UpperCAmelCase       :							Any  = list(temp.keys()			)
        for key in keys:
          UpperCAmelCase       :							Union[str, Any]  = temp.pop(__snake_case			)

        if tensors is None:
          UpperCAmelCase       :							Dict  = temp
        else:
          for key in tensors.keys():
            if any(key.endswith(__snake_case			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              UpperCAmelCase       :							Tuple  = 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
              # We concatenate these weights accross TP ranks
              UpperCAmelCase       :							Any  = torch.cat([tensors[key], temp[key]]				, dim=__snake_case			)

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(__snake_case			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
          UpperCAmelCase       :							str  = tensors[key] / pretraining_tp
      torch.save(
          __snake_case				, os.path.join(
              __snake_case				, '''pytorch_model_{}-of-{}.bin'''.format(str(j + 1			).zfill(5			)				, str(len(__snake_case			)			).zfill(5			)			)				, )				, )

      for key in tensors.keys():
        UpperCAmelCase       :							Any  = tensors[key]
        total_size += value.numel() * get_dtype_size(value.dtype			)
        if key not in index_dict["weight_map"]:
          UpperCAmelCase       :							Optional[Any]  = '''pytorch_model_{}-of-{}.bin'''.format(
              str(j + 1			).zfill(5			)				, str(len(__snake_case			)			).zfill(5			)			)

    UpperCAmelCase       :							str  = BloomConfig()
    UpperCAmelCase       :							Optional[int]  = pytorch_dump_folder_path + '''/''' + CONFIG_NAME
    UpperCAmelCase       :							Any  = total_size
    with open(__snake_case				, '''w'''				, encoding='''utf-8'''			) as f:
      f.write(config.to_json_string()			)
    with open(os.path.join(__snake_case				, WEIGHTS_NAME + '''.index.json'''			)				, '''w'''				, encoding='''utf-8'''			) as f:
      UpperCAmelCase       :							Union[str, Any]  = json.dumps(__snake_case				, indent=2				, sort_keys=__snake_case			) + '''\n'''
      f.write(__snake_case			)
  else:
    UpperCAmelCase       :							Dict  = BloomModel(__snake_case			)

    UpperCAmelCase       :							Any  = os.listdir(__snake_case			)
    UpperCAmelCase       :							int  = sorted(filter(lambda UpperCAmelCase			: s.startswith('''layer'''			) and "model_00" in s				, __snake_case			)			)

    UpperCAmelCase       :							Any  = None
    for i, file in enumerate(__snake_case			):
      UpperCAmelCase       :							Optional[int]  = None
      for i in range(__snake_case			):
        # load all TP files
        UpperCAmelCase       :							Optional[Any]  = file.replace('''model_00'''				, f'''model_0{i}'''			)
        UpperCAmelCase       :							List[Any]  = torch.load(os.path.join(__snake_case				, __snake_case			)				, map_location='''cpu'''			)

        # Rename keys in the transformers names
        UpperCAmelCase       :							int  = list(temp.keys()			)
        for key in keys:
          UpperCAmelCase       :							Union[str, Any]  = temp.pop(__snake_case			)

        if tensors is None:
          UpperCAmelCase       :							Tuple  = temp
        else:
          for key in tensors.keys():
            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
            if any(key.endswith(__snake_case			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              UpperCAmelCase       :							List[Any]  = 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
              # We concatenate these weights accross TP ranks
              UpperCAmelCase       :							List[Any]  = torch.cat([tensors[key], temp[key]]				, dim=__snake_case			)

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(__snake_case			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
          UpperCAmelCase       :							Dict  = tensors[key] / pretraining_tp

      UpperCAmelCase       :							List[str]  = model.load_state_dict(__snake_case				, strict=__snake_case			)
      assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
      if missing_keys is None:
        UpperCAmelCase       :							Tuple  = set(other_keys.missing_keys			)
      else:
        UpperCAmelCase       :							List[Any]  = missing_keys.intersection(set(other_keys.missing_keys			)			)

    assert not missing_keys, f'''The keys {missing_keys} are missing'''

    # Save pytorch-model
    os.makedirs(__snake_case				, exist_ok=__snake_case			)
    UpperCAmelCase       :							List[str]  = pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
    UpperCAmelCase       :							Dict  = pytorch_dump_folder_path + '''/''' + CONFIG_NAME
    print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''			)
    if config.torch_dtype is not None:
      UpperCAmelCase       :							Any  = model.to(config.torch_dtype			)
    torch.save(model.state_dict()				, __snake_case			)
    print(f'''Save configuration file to {pytorch_config_dump_path}'''			)
    with open(__snake_case				, '''w'''				, encoding='''utf-8'''			) as f:
      f.write(config.to_json_string()			)


if __name__ == "__main__":
     _lowerCamelCase				:   int					   =							argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         "--bloom_checkpoint_path",
         default=None,
         type=str,
         required=True,
         help="Path to the Megatron-LM checkpoint path.",
     )
     parser.add_argument(
         "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
     )
     parser.add_argument(
         "--bloom_config_file",
         default="",
         type=str,
         help=(
             "An optional config json file corresponding to the pre-trained model. \n"
             "This specifies the model architecture."
         ),
     )
     parser.add_argument(
         "--shard_model",
         action="store_true",
         help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
     )
     parser.add_argument(
         "--pretraining_tp",
         default=4,
         type=int,
         help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
     )
     _lowerCamelCase				:   str					   =							parser.parse_args()
     convert_bloom_checkpoint_to_pytorch(
         args.bloom_checkpoint_path,
         args.bloom_config_file,
         args.pytorch_dump_folder_path,
         args.shard_model,
         args.pretraining_tp,
     )







"""simple docstring"""





import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


_a														=		"""http://www.mocksite.com/file1.txt"""
_a														=		"""\"text\": [\"foo\", \"foo\"]"""
_a														=		"""6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"""






class     _UpperCAmelCase:
		lowercase__						=      2_00
		lowercase__						=      {'Content-Length': '100'}
		lowercase__						=      {}



		def 						UpperCAmelCase							(					self				,		**__a)       ->       Optional[int]:






							'''simple docstring'''
							return [bytes(__a				,		'''utf-8''')]





def       lowerCamelCase__		(			*__snake_case,     **__snake_case	)							->		int:


					"""simple docstring"""





					return MockResponse()



@pytest.mark.parametrize('''urls_type''',     [str, list, dict]	)
def       lowerCamelCase__		(			__snake_case,     __snake_case,     __snake_case	)							->		Dict:


					"""simple docstring"""





					import requests

					monkeypatch.setattr(__snake_case,     '''request''',     __snake_case	)

					_UpperCamelCase							 =  URL
					if issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  url
					elif issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  [url]
					elif issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  {'''train''': url}
					_UpperCamelCase							 =  '''dummy'''
					_UpperCamelCase							 =  '''downloads'''
					_UpperCamelCase							 =  tmp_path
					_UpperCamelCase							 =  DownloadConfig(
					    cache_dir=os.path.join(__snake_case,     __snake_case	),     use_etag=__snake_case,     )
					_UpperCamelCase							 =  DownloadManager(dataset_name=__snake_case,     download_config=__snake_case	)
					_UpperCamelCase							 =  dl_manager.download(__snake_case	)
					_UpperCamelCase							 =  urls
					for downloaded_paths in [downloaded_paths]:
										if isinstance(__snake_case,     __snake_case	):
															_UpperCamelCase							 =  [downloaded_paths]
															_UpperCamelCase							 =  [urls]
										elif isinstance(__snake_case,     __snake_case	):
															assert "train" in downloaded_paths.keys()
															_UpperCamelCase							 =  downloaded_paths.values()
															_UpperCamelCase							 =  urls.values()
										assert downloaded_paths
										for downloaded_path, input_url in zip(__snake_case,     __snake_case	):
															assert downloaded_path == dl_manager.downloaded_paths[input_url]
															_UpperCamelCase							 =  Path(__snake_case	)
															_UpperCamelCase							 =  downloaded_path.parts
															assert parts[-1] == HASH
															assert parts[-2] == cache_subdir
															assert downloaded_path.exists()
															_UpperCamelCase							 =  downloaded_path.read_text()
															assert content == CONTENT
															_UpperCamelCase							 =  downloaded_path.with_suffix('''.json'''	)
															assert metadata_downloaded_path.exists()
															_UpperCamelCase							 =  json.loads(metadata_downloaded_path.read_text()	)
															assert metadata_content == {"url": URL, "etag": None}



@pytest.mark.parametrize('''paths_type''',     [str, list, dict]	)
def       lowerCamelCase__		(			__snake_case,     __snake_case,     __snake_case	)							->		List[str]:


					"""simple docstring"""





					_UpperCamelCase							 =  str(__snake_case	)
					if issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  filename
					elif issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  [filename]
					elif issubclass(__snake_case,     __snake_case	):
										_UpperCamelCase							 =  {'''train''': filename}
					_UpperCamelCase							 =  '''dummy'''
					_UpperCamelCase							 =  xz_file.parent
					_UpperCamelCase							 =  '''extracted'''
					_UpperCamelCase							 =  DownloadConfig(
					    cache_dir=__snake_case,     use_etag=__snake_case,     )
					_UpperCamelCase							 =  DownloadManager(dataset_name=__snake_case,     download_config=__snake_case	)
					_UpperCamelCase							 =  dl_manager.extract(__snake_case	)
					_UpperCamelCase							 =  paths
					for extracted_paths in [extracted_paths]:
										if isinstance(__snake_case,     __snake_case	):
															_UpperCamelCase							 =  [extracted_paths]
															_UpperCamelCase							 =  [paths]
										elif isinstance(__snake_case,     __snake_case	):
															assert "train" in extracted_paths.keys()
															_UpperCamelCase							 =  extracted_paths.values()
															_UpperCamelCase							 =  paths.values()
										assert extracted_paths
										for extracted_path, input_path in zip(__snake_case,     __snake_case	):
															assert extracted_path == dl_manager.extracted_paths[input_path]
															_UpperCamelCase							 =  Path(__snake_case	)
															_UpperCamelCase							 =  extracted_path.parts
															assert parts[-1] == hash_url_to_filename(__snake_case,     etag=__snake_case	)
															assert parts[-2] == extracted_subdir
															assert extracted_path.exists()
															_UpperCamelCase							 =  extracted_path.read_text()
															_UpperCamelCase							 =  text_file.read_text()
															assert extracted_file_content == expected_file_content



def       lowerCamelCase__		(			__snake_case,     __snake_case	)							->		Optional[Any]:


					"""simple docstring"""





					assert path.endswith('''.jsonl'''	)
					for num_items, line in enumerate(__snake_case,     start=1	):
										_UpperCamelCase							 =  json.loads(line.decode('''utf-8'''	)	)
										assert item.keys() == {"col_1", "col_2", "col_3"}
					assert num_items == 4



@pytest.mark.parametrize('''archive_jsonl''',     ['''tar_jsonl_path''', '''zip_jsonl_path''']	)
def       lowerCamelCase__		(			__snake_case,     __snake_case	)							->		int:


					"""simple docstring"""





					_UpperCamelCase							 =  request.getfixturevalue(__snake_case	)
					_UpperCamelCase							 =  DownloadManager()
					for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(__snake_case	),     start=1	):
										_test_jsonl(__snake_case,     __snake_case	)
					assert num_jsonl == 2



@pytest.mark.parametrize('''archive_nested_jsonl''',     ['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']	)
def       lowerCamelCase__		(			__snake_case,     __snake_case	)							->		int:


					"""simple docstring"""





					_UpperCamelCase							 =  request.getfixturevalue(__snake_case	)
					_UpperCamelCase							 =  DownloadManager()
					for num_tar, (path, file) in enumerate(dl_manager.iter_archive(__snake_case	),     start=1	):
										for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(__snake_case	),     start=1	):
															_test_jsonl(__snake_case,     __snake_case	)
					assert num_tar == 1
					assert num_jsonl == 2



def       lowerCamelCase__		(			__snake_case	)							->		Dict:


					"""simple docstring"""





					_UpperCamelCase							 =  DownloadManager()
					for num_file, file in enumerate(dl_manager.iter_files(__snake_case	),     start=1	):
										assert os.path.basename(__snake_case	) == ("test.txt" if num_file == 1 else "train.txt")
					assert num_file == 2






