



'''simple docstring'''






#
# Sort large text files in a minimum amount of memory
#
import argparse
import os

class 	_lowerCAmelCase     :






		'''simple docstring'''
		snake_case_							= 'block_{0}.dat'
		def __init__(						self							:       Dict      ,		UpperCamelCase_							:       Any		)      ->			Tuple:







									'''simple docstring'''





									_lowercase							:		Any				=   filename
									_lowercase							:		List[str]				=   []
		def 						__lowercase      (						self							:       str      ,		UpperCamelCase_							:       Any      ,		UpperCamelCase_							:       Any		)      ->			Optional[int]:







									'''simple docstring'''





									_lowercase							:		List[str]				=   self.BLOCK_FILENAME_FORMAT.format(UpperCamelCase_		)
									with open(UpperCamelCase_      ,		'''w'''		) as file:
																file.write(UpperCamelCase_		)
									self.block_filenames.append(UpperCamelCase_		)
		def 						__lowercase      (						self							:       str		)      ->			List[Any]:







									'''simple docstring'''





									return self.block_filenames
		def 						__lowercase      (						self							:       Tuple      ,		UpperCamelCase_							:       List[str]      ,		UpperCamelCase_							:       List[Any]=None		)      ->			Tuple:







									'''simple docstring'''





									_lowercase							:		Union[str, Any]				=   0
									with open(self.filename		) as file:
																while True:
																							_lowercase							:		Optional[Any]				=   file.readlines(UpperCamelCase_		)

																							if lines == []:
																														break

																							if sort_key is None:
																														lines.sort()
																							else:
																														lines.sort(key=UpperCamelCase_		)

																							self.write_block(''''''.join(UpperCamelCase_		)      ,		UpperCamelCase_		)
																							i += 1





		def 						__lowercase      (						self							:       int		)      ->			Tuple:







									'''simple docstring'''





									map(os.remove      ,		self.block_filenames		)




class 	_lowerCAmelCase     :






		'''simple docstring'''
		def 						__lowercase      (						self							:       Union[str, Any]      ,		UpperCamelCase_							:       List[str]		)      ->			int:







									'''simple docstring'''





									_lowercase							:		Union[str, Any]				=   -1
									_lowercase							:		Dict				=   None

									for i in range(len(UpperCamelCase_		)		):
																if min_str is None or choices[i] < min_str:
																							_lowercase							:		Dict				=   i

									return min_index




class 	_lowerCAmelCase     :






		'''simple docstring'''
		def __init__(						self							:       Union[str, Any]      ,		UpperCamelCase_							:       List[str]		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Dict				=   files
									_lowercase							:		Optional[int]				=   set()
									_lowercase							:		List[str]				=   len(UpperCamelCase_		)
									_lowercase							:		Union[str, Any]				=   {i: None for i in range(self.num_buffers		)}
		def 						__lowercase      (						self							:       Tuple		)      ->			List[str]:







									'''simple docstring'''





									return {
									    i: self.buffers[i] for i in range(self.num_buffers		) if i not in self.empty
									}
		def 						__lowercase      (						self							:       int		)      ->			List[Any]:







									'''simple docstring'''





									for i in range(self.num_buffers		):
																if self.buffers[i] is None and i not in self.empty:
																							_lowercase							:		Dict				=   self.files[i].readline()

																							if self.buffers[i] == "":
																														self.empty.add(UpperCamelCase_		)
																														self.files[i].close()

									if len(self.empty		) == self.num_buffers:
																return False

									return True





		def 						__lowercase      (						self							:       str      ,		UpperCamelCase_							:       List[str]		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Union[str, Any]				=   self.buffers[index]
									_lowercase							:		Union[str, Any]				=   None

									return value




class 	_lowerCAmelCase     :






		'''simple docstring'''
		def __init__(						self							:       List[str]      ,		UpperCamelCase_							:       Any		)      ->			Optional[int]:







									'''simple docstring'''





									_lowercase							:		str				=   merge_strategy
		def 						__lowercase      (						self							:       Tuple      ,		UpperCamelCase_							:       Optional[int]      ,		UpperCamelCase_							:       Any      ,		UpperCamelCase_							:       int		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Dict				=   FilesArray(self.get_file_handles(UpperCamelCase_      ,		UpperCamelCase_		)		)
									with open(UpperCamelCase_      ,		'''w'''      ,		UpperCamelCase_		) as outfile:
																while buffers.refresh():
																							_lowercase							:		Tuple				=   self.merge_strategy.select(buffers.get_dict()		)
																							outfile.write(buffers.unshift(UpperCamelCase_		)		)





		def 						__lowercase      (						self							:       Dict      ,		UpperCamelCase_							:       int      ,		UpperCamelCase_							:       Optional[Any]		)      ->			Union[str, Any]:







									'''simple docstring'''





									_lowercase							:		Tuple				=   {}

									for i in range(len(UpperCamelCase_		)		):
																_lowercase							:		Optional[int]				=   open(filenames[i]      ,		'''r'''      ,		UpperCamelCase_		)  # noqa: UP015

									return files




class 	_lowerCAmelCase     :






		'''simple docstring'''
		def __init__(						self							:       Any      ,		UpperCamelCase_							:       Any		)      ->			Optional[int]:







									'''simple docstring'''





									_lowercase							:		Dict				=   block_size
		def 						__lowercase      (						self							:       Dict      ,		UpperCamelCase_							:       Any      ,		UpperCamelCase_							:       Optional[int]=None		)      ->			Tuple:







									'''simple docstring'''





									_lowercase							:		int				=   self.get_number_blocks(UpperCamelCase_      ,		self.block_size		)
									_lowercase							:		Optional[int]				=   FileSplitter(UpperCamelCase_		)
									splitter.split(self.block_size      ,		UpperCamelCase_		)

									_lowercase							:		List[str]				=   FileMerger(NWayMerge()		)
									_lowercase							:		Dict				=   self.block_size / (num_blocks + 1)
									merger.merge(splitter.get_block_filenames()      ,		filename + '''.out'''      ,		UpperCamelCase_		)

									splitter.cleanup()





		def 						__lowercase      (						self							:       List[Any]      ,		UpperCamelCase_							:       Dict      ,		UpperCamelCase_							:       Optional[Any]		)      ->			int:







									'''simple docstring'''





									return (os.stat(UpperCamelCase_		).st_size / block_size) + 1





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     Optional[int]  )		->List[Any]:






							'''simple docstring'''
							if string[-1].lower() == "k":
														return int(string[:-1]  ) * 10_24
							elif string[-1].lower() == "m":
														return int(string[:-1]  ) * 10_24 * 10_24
							elif string[-1].lower() == "g":
														return int(string[:-1]  ) * 10_24 * 10_24 * 10_24
							else:
														return int(snake_case_  )





def 				_SCREAMING_SNAKE_CASE(					)		->Optional[Any]:






							'''simple docstring'''
							_lowercase							:		int				=   argparse.ArgumentParser()
							parser.add_argument(
							    '''-m''' ,       '''--mem''' ,       help='''amount of memory to use for sorting''' ,       default='''100M'''  )
							parser.add_argument(
							    '''filename''' ,       metavar='''<filename>''' ,       nargs=1 ,       help='''name of file to sort'''  )
							_lowercase							:		int				=   parser.parse_args()

							_lowercase							:		Optional[Any]				=   ExternalSort(parse_memory(args.mem  )  )
							sorter.sort(args.filename[0]  )


if __name__ == "__main__":
						main()








'''simple docstring'''






import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
						import torch


lowerCamelCase__	  =		logging.get_logger(__name__)

class 	_lowerCAmelCase     :






		'''simple docstring'''
		def __init__(						self							:       Optional[int]      ,		UpperCamelCase_							:       Optional[Any]      ,		UpperCamelCase_							:       Union[str, Any]=21      ,		UpperCamelCase_							:       Dict=[]      ,		UpperCamelCase_							:       Any=None		)      ->			Any:







									'''simple docstring'''





									_lowercase							:		Optional[Any]				=   model
									_lowercase							:		Tuple				=   trace_batch_nums
									_lowercase							:		Optional[Any]				=   abort_after_batch_num

									# keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
									_lowercase							:		int				=   collections.deque([]      ,		UpperCamelCase_		)
									_lowercase							:		List[Any]				=   []
									_lowercase							:		List[str]				=   0
									_lowercase							:		Optional[int]				=   0
									_lowercase							:		List[Any]				=   False
									_lowercase							:		List[Any]				=   '''                 '''

									self.analyse_model()

									self.register_forward_hook()
		def 						__lowercase      (						self							:       Union[str, Any]      ,		UpperCamelCase_							:       Optional[Any]=None		)      ->			Optional[Any]:







									'''simple docstring'''





									if frame is not None:
																self.expand_frame(UpperCamelCase_		)
									self.frames.append('''\n'''.join(self.frame		)		)
									_lowercase							:		Dict				=   []  # start a new frame
		def 						__lowercase      (						self							:       Union[str, Any]      ,		UpperCamelCase_							:       Optional[Any]		)      ->			Union[str, Any]:







									'''simple docstring'''





									self.frame.append(UpperCamelCase_		)
		def 						__lowercase      (						self							:       str		)      ->			List[str]:







									'''simple docstring'''





									print('''\n'''.join(self.frames		)		)
									_lowercase							:		Union[str, Any]				=   []
		def 						__lowercase      (						self							:       Optional[int]		)      ->			List[str]:







									'''simple docstring'''





									_lowercase							:		Optional[Any]				=   []
		def 						__lowercase      (						self							:       List[Any]		)      ->			Optional[int]:







									'''simple docstring'''





									print(F"\nDetected inf/nan during batch_number={self.batch_number}"		)
									print(F"Last {len(self.frames		)} forward frames:"		)
									print(F"{'abs min':8} {'abs max':8} metadata"		)
									print('''\n'''.join(self.frames		)		)
									print('''\n\n'''		)
									_lowercase							:		List[Any]				=   []
		def 						__lowercase      (						self							:       List[str]		)      ->			Tuple:







									'''simple docstring'''





									_lowercase							:		int				=   {m: name for name, m in self.model.named_modules()}
									# self.longest_module_name = max(len(v) for v in self.module_names.values())
		def 						__lowercase      (						self							:       Dict      ,		UpperCamelCase_							:       str      ,		UpperCamelCase_							:       Optional[int]		)      ->			int:







									'''simple docstring'''





									if torch.is_tensor(UpperCamelCase_		):
																self.expand_frame(get_abs_min_max(UpperCamelCase_      ,		UpperCamelCase_		)		)
																if detect_overflow(UpperCamelCase_      ,		UpperCamelCase_		):
																							_lowercase							:		Any				=   True
									elif var is None:
																self.expand_frame(F"{'None':>17} {ctx}"		)
									else:
																self.expand_frame(F"{'not a tensor':>17} {ctx}"		)
		def 						__lowercase      (						self							:       Union[str, Any]		)      ->			int:







									'''simple docstring'''





									self.expand_frame(F"\n\n{self.prefix} *** Starting batch number={self.batch_number} ***"		)
									self.expand_frame(F"{'abs min':8} {'abs max':8} metadata"		)
		def 						__lowercase      (						self							:       str		)      ->			Any:







									'''simple docstring'''





									self.expand_frame(F"{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n"		)
		def 						__lowercase      (						self							:       Tuple      ,		UpperCamelCase_							:       Any      ,		UpperCamelCase_							:       Union[str, Any]      ,		UpperCamelCase_							:       Optional[int]		)      ->			Dict:







									'''simple docstring'''





									self.expand_frame(F"{self.prefix} {self.module_names[module]} {module.__class__.__name__}"		)

									# params
									for name, p in module.named_parameters(recurse=UpperCamelCase_		):
																self.analyse_variable(UpperCamelCase_      ,		UpperCamelCase_		)

									# inputs
									if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																for i, x in enumerate(UpperCamelCase_		):
																							self.analyse_variable(UpperCamelCase_      ,		F"input[{i}]"		)
									else:
																self.analyse_variable(UpperCamelCase_      ,		'''input'''		)

									# outputs
									if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																for i, x in enumerate(UpperCamelCase_		):
																							# possibly a tuple of tuples
																							if isinstance(UpperCamelCase_      ,		UpperCamelCase_		):
																														for j, y in enumerate(UpperCamelCase_		):
																																					self.analyse_variable(UpperCamelCase_      ,		F"output[{i}][{j}]"		)
																							else:
																														self.analyse_variable(UpperCamelCase_      ,		F"output[{i}]"		)
									else:
																self.analyse_variable(UpperCamelCase_      ,		'''output'''		)

									self.save_frame()
		def 						__lowercase      (						self							:       Optional[int]		)      ->			Dict:







									'''simple docstring'''





									self.model.apply(self._register_forward_hook		)
		def 						__lowercase      (						self							:       Optional[int]      ,		UpperCamelCase_							:       int		)      ->			List[str]:







									'''simple docstring'''





									module.register_forward_hook(self.forward_hook		)





		def 						__lowercase      (						self							:       Optional[int]      ,		UpperCamelCase_							:       Optional[int]      ,		UpperCamelCase_							:       Dict      ,		UpperCamelCase_							:       List[Any]		)      ->			List[Any]:







									'''simple docstring'''





									_lowercase							:		str				=   False

									_lowercase							:		Dict				=   True if self.batch_number in self.trace_batch_nums else False
									if trace_mode:
																self.reset_saved_frames()

									if self.total_calls == 0:
																self.batch_start_frame()
									self.total_calls += 1

									# count batch numbers - the very first forward hook of the batch will be called when the
									# batch completes - i.e. it gets called very last - we know this batch has finished
									if module == self.model:
																self.batch_number += 1
																_lowercase							:		Union[str, Any]				=   True

									self.create_frame(UpperCamelCase_      ,		UpperCamelCase_      ,		UpperCamelCase_		)

									# if last_frame_of_batch:
									#     self.batch_end_frame()

									if trace_mode:
																self.trace_frames()

									if last_frame_of_batch:
																self.batch_start_frame()

									if self.detected_overflow and not trace_mode:
																self.dump_saved_frames()

																# now we can abort, as it's pointless to continue running
																raise ValueError(
																    '''DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '''
																    '''Please scroll up above this traceback to see the activation values prior to this event.'''		)

									# abort after certain batch if requested to do so
									if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
																raise ValueError(
																    F"DebugUnderflowOverflow: aborting after {self.batch_number} batches due to"
																    F" `abort_after_batch_num={self.abort_after_batch_num}` arg"		)





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     Any ,       snake_case_    :     Dict  )		->Any:






							'''simple docstring'''
							_lowercase							:		Union[str, Any]				=   var.abs()
							return F"{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}"





def 				_SCREAMING_SNAKE_CASE(					snake_case_    :     int ,       snake_case_    :     Any  )		->int:






							'''simple docstring'''
							_lowercase							:		List[Any]				=   False
							if torch.isnan(snake_case_  ).any().item():
														_lowercase							:		Any				=   True
														print(F"{ctx} has nans"  )
							if torch.isinf(snake_case_  ).any().item():
														_lowercase							:		Optional[Any]				=   True
														print(F"{ctx} has infs"  )

							# if needed to monitor large elements can enable the following
							if 0:  # and detected:
														_lowercase							:		List[Any]				=   var[torch.ge(var.abs() ,       1_00  )]
														if naaa.numel() > 0:
																					print(F"{ctx}:  n100={naaa.numel()}"  )
														_lowercase							:		List[str]				=   var[torch.ge(var.abs() ,       10_00  )]
														if naaaa.numel() > 0:
																					print(F"{ctx}: n1000={naaaa.numel()}"  )
														_lowercase							:		Tuple				=   var[torch.ge(var.abs() ,       1_00_00  )]
														if naaaaa.numel() > 0:
																					print(F"{ctx}: n10000={naaaaa.numel()}"  )

							if 0:
														print(F"min={var.min():9.2e} max={var.max():9.2e}"  )

							if 0:
														print(F"min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})"  )

							return detected




class 	_lowerCAmelCase     (    __A       ):






		'''simple docstring'''
		snake_case_							= 'underflow_overflow'
		snake_case_							= 'tpu_metrics_debug'



