
#
# Sort large text files in a minimum amount of memory
#
import argparse
import os





class      UpperCamelCase_			:







						'''simple docstring'''



						UpperCamelCase			:     int							      =				'''block_{0}.dat'''





						def __init__( self       :str		,						lowerCAmelCase__       :Any    )     ->List[Any]:
										lowercase						    =			filename
										lowercase						    =			[]





						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :Any    )     ->str:
										lowercase						    =			self.BLOCK_FILENAME_FORMAT.format(lowerCAmelCase__    )
										with open(lowerCAmelCase__		,						"w"    ) as file:
														file.write(lowerCAmelCase__    )
										self.block_filenames.append(lowerCAmelCase__    )





						def 					SCREAMING_SNAKE_CASE( self       :Dict    )     ->Any:
										return self.block_filenames





						def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :int=None    )     ->Tuple:
										lowercase						    =			0
										with open(self.filename    ) as file:
														while True:
																		lowercase						    =			file.readlines(lowerCAmelCase__    )

																		if lines == []:
																						break

																		if sort_key is None:
																						lines.sort()
																		else:
																						lines.sort(key=lowerCAmelCase__    )

																		self.write_block("".join(lowerCAmelCase__    )		,						lowerCAmelCase__    )
																		i += 1





						def 					SCREAMING_SNAKE_CASE( self       :Tuple    )     ->Optional[int]:
										map(os.remove		,						self.block_filenames    )





class      UpperCamelCase_			:







						'''simple docstring'''



						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :Optional[int]    )     ->List[str]:
										lowercase						    =			-1
										lowercase						    =			None

										for i in range(len(lowerCAmelCase__    )    ):
														if min_str is None or choices[i] < min_str:
																		lowercase						    =			i

										return min_index





class      UpperCamelCase_			:







						'''simple docstring'''



						def __init__( self       :List[Any]		,						lowerCAmelCase__       :List[Any]    )     ->Tuple:
										lowercase						    =			files
										lowercase						    =			set()
										lowercase						    =			len(lowerCAmelCase__    )
										lowercase						    =			{i: None for i in range(self.num_buffers    )}





						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]    )     ->Optional[Any]:
										return {
										    i: self.buffers[i] for i in range(self.num_buffers    ) if i not in self.empty
										}





						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]    )     ->Union[str, Any]:
										for i in range(self.num_buffers    ):
														if self.buffers[i] is None and i not in self.empty:
																		lowercase						    =			self.files[i].readline()

																		if self.buffers[i] == "":
																						self.empty.add(lowerCAmelCase__    )
																						self.files[i].close()

										if len(self.empty    ) == self.num_buffers:
														return False

										return True





						def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :str    )     ->Tuple:
										lowercase						    =			self.buffers[index]
										lowercase						    =			None

										return value





class      UpperCamelCase_			:







						'''simple docstring'''



						def __init__( self       :List[str]		,						lowerCAmelCase__       :Any    )     ->List[str]:
										lowercase						    =			merge_strategy





						def 					SCREAMING_SNAKE_CASE( self       :Optional[int]		,						lowerCAmelCase__       :Any		,						lowerCAmelCase__       :List[str]		,						lowerCAmelCase__       :Tuple    )     ->str:
										lowercase						    =			FilesArray(self.get_file_handles(lowerCAmelCase__		,						lowerCAmelCase__    )    )
										with open(lowerCAmelCase__		,						"w"		,						lowerCAmelCase__    ) as outfile:
														while buffers.refresh():
																		lowercase						    =			self.merge_strategy.select(buffers.get_dict()    )
																		outfile.write(buffers.unshift(lowerCAmelCase__    )    )





						def 					SCREAMING_SNAKE_CASE( self       :List[str]		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :Any    )     ->int:
										lowercase						    =			{}

										for i in range(len(lowerCAmelCase__    )    ):
														lowercase						    =			open(filenames[i]		,						"r"		,						lowerCAmelCase__    )  # noqa: UP015

										return files





class      UpperCamelCase_			:







						'''simple docstring'''



						def __init__( self       :str		,						lowerCAmelCase__       :List[str]    )     ->Tuple:
										lowercase						    =			block_size





						def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :Union[str, Any]		,						lowerCAmelCase__       :List[str]=None    )     ->Optional[Any]:
										lowercase						    =			self.get_number_blocks(lowerCAmelCase__		,						self.block_size    )
										lowercase						    =			FileSplitter(lowerCAmelCase__    )
										splitter.split(self.block_size		,						lowerCAmelCase__    )

										lowercase						    =			FileMerger(NWayMerge()    )
										lowercase						    =			self.block_size / (num_blocks + 1)
										merger.merge(splitter.get_block_filenames()		,						filename + ".out"		,						lowerCAmelCase__    )

										splitter.cleanup()





						def 					SCREAMING_SNAKE_CASE( self       :List[str]		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Optional[int]    )     ->List[str]:
										return (os.stat(lowerCAmelCase__    ).st_size / block_size) + 1




def    __snake_case       (  __magic_name__	):







				'''simple docstring'''


				if string[-1].lower() == "k":
								return int(string[:-1]	) * 1024
				elif string[-1].lower() == "m":
								return int(string[:-1]	) * 1024 * 1024
				elif string[-1].lower() == "g":
								return int(string[:-1]	) * 1024 * 1024 * 1024
				else:
								return int(__magic_name__	)

def    __snake_case       (  ):







				'''simple docstring'''


				lowercase						    =			argparse.ArgumentParser()
				parser.add_argument(
				    "-m"		,						"--mem"		,						help="amount of memory to use for sorting"		,						default="100M"	)
				parser.add_argument(
				    "filename"		,						metavar="<filename>"		,						nargs=1		,						help="name of file to sort"	)
				lowercase						    =			parser.parse_args()

				lowercase						    =			ExternalSort(parse_memory(args.mem	)	)
				sorter.sort(args.filename[0]	)


if __name__ == "__main__":
						main()


# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


_snake_case			:       Optional[Any]      =						logging.get_logger(__name__)


if is_torch_available():
						import torch
						from torch import nn
						from torch.utils.data import Dataset

						from transformers import Trainer





						class      UpperCamelCase_			(			__a		):







												'''simple docstring'''



												def __init__( self       :Optional[int]		,						lowerCAmelCase__       :int = 101    )     ->Any:
																lowercase						    =			length





												def __len__( self       :Tuple    )     ->Any:
																return self.length





												def __getitem__( self       :Any		,						lowerCAmelCase__       :Tuple    )     ->int:
																return i





						class      UpperCamelCase_			:







												'''simple docstring'''



												def __call__( self       :Optional[Any]		,						lowerCAmelCase__       :List[Any]    )     ->int:
																return {"input_ids": torch.tensor(lowerCAmelCase__    ), "labels": torch.tensor(lowerCAmelCase__    )}





						class      UpperCamelCase_			(			nn.Module		):







												'''simple docstring'''



												def __init__( self       :Any    )     ->str:
																super().__init__()
																# Add some (unused) params otherwise DDP will complain.
																lowercase						    =			nn.Linear(120		,						80    )





												def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :Optional[int]		,						lowerCAmelCase__       :Union[str, Any]=None    )     ->List[Any]:
																if labels is not None:
																				return torch.tensor(0.0		,						device=input_ids.device    ), input_ids
																else:
																				return input_ids

def    __snake_case       (  ):







				'''simple docstring'''


				lowercase						    =			HfArgumentParser((TrainingArguments,)	)
				sys.argv += ["--output_dir", "./examples"]
				lowercase						    =			parser.parse_args_into_dataclasses()[0]

				logger.warning(
				    F'''Process rank: {training_args.local_rank}, device: {training_args.device}, '''
				    F'''tpu_num_cores: {training_args.tpu_num_cores}'''		,						)

				# Essentially, what we want to verify in the distributed case is
				# that we get all samples back, in the right order.
				# (this is crucial for prediction for instance)
				for dataset_length in [1001, 256, 15]:
								lowercase						    =			DummyDataset(__magic_name__	)

								def compute_metrics(__magic_name__	) -> Dict:
												lowercase						    =			list(range(len(__magic_name__	)	)	)
												lowercase						    =			p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
												return {"success": success}

								lowercase						    =			Trainer(
								    model=DummyModel()		,						args=__magic_name__		,						data_collator=DummyDataCollator()		,						eval_dataset=__magic_name__		,						compute_metrics=__magic_name__		,						)
								lowercase						    =			trainer.evaluate()
								logger.info(__magic_name__	)
								if metrics["eval_success"] is not True:
												logger.error(__magic_name__	)
												exit(1	)

								lowercase						    =			trainer.predict(__magic_name__	)
								logger.info(p.metrics	)
								if p.metrics["test_success"] is not True:
												logger.error(p.metrics	)
												exit(1	)

								lowercase						    =			2

								lowercase						    =			trainer.evaluate()
								logger.info(__magic_name__	)
								if metrics["eval_success"] is not True:
												logger.error(__magic_name__	)
												exit(1	)

								lowercase						    =			trainer.predict(__magic_name__	)
								logger.info(p.metrics	)
								if p.metrics["test_success"] is not True:
												logger.error(p.metrics	)
												exit(1	)

								lowercase						    =			None

				logger.info("🔥 All distributed tests successful"	)

def    __snake_case       (  __magic_name__	):







				'''simple docstring'''


				main()


if __name__ == "__main__":
						main()
