
"""simple docstring"""






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
__UpperCAmelCase       =  logging.getLogger(__name__)






def 						_snake_case       (	)  ->				Optional[Any]:


			'''simple docstring'''





			lowerCAmelCase_    :List[Any]			     =  argparse.ArgumentParser(
			    description="""Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."""		)
			parser.add_argument("""--file_path"""      ,     type=lowercase__      ,     default="""data/dump.txt"""      ,     help="""The path to the data."""		)
			parser.add_argument("""--tokenizer_type"""      ,     type=lowercase__      ,     default="""bert"""      ,     choices=["""bert""", """roberta""", """gpt2"""]		)
			parser.add_argument("""--tokenizer_name"""      ,     type=lowercase__      ,     default="""bert-base-uncased"""      ,     help="""The tokenizer to use."""		)
			parser.add_argument("""--dump_file"""      ,     type=lowercase__      ,     default="""data/dump"""      ,     help="""The dump file prefix."""		)
			lowerCAmelCase_    :Any			     =  parser.parse_args()

			logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""		)
			if args.tokenizer_type == "bert":
						lowerCAmelCase_    :Tuple			     =  BertTokenizer.from_pretrained(args.tokenizer_name		)
						lowerCAmelCase_    :Optional[int]			     =  tokenizer.special_tokens_map["""cls_token"""]  # `[CLS]`
						lowerCAmelCase_    :Optional[Any]			     =  tokenizer.special_tokens_map["""sep_token"""]  # `[SEP]`
			elif args.tokenizer_type == "roberta":
						lowerCAmelCase_    :Any			     =  RobertaTokenizer.from_pretrained(args.tokenizer_name		)
						lowerCAmelCase_    :str			     =  tokenizer.special_tokens_map["""cls_token"""]  # `<s>`
						lowerCAmelCase_    :Dict			     =  tokenizer.special_tokens_map["""sep_token"""]  # `</s>`
			elif args.tokenizer_type == "gpt2":
						lowerCAmelCase_    :Optional[int]			     =  GPTaTokenizer.from_pretrained(args.tokenizer_name		)
						lowerCAmelCase_    :Any			     =  tokenizer.special_tokens_map["""bos_token"""]  # `<|endoftext|>`
						lowerCAmelCase_    :Dict			     =  tokenizer.special_tokens_map["""eos_token"""]  # `<|endoftext|>`

			logger.info(f"""Loading text from {args.file_path}"""		)
			with open(args.file_path      ,     """r"""      ,     encoding="""utf8"""		) as fp:
						lowerCAmelCase_    :List[Any]			     =  fp.readlines()

			logger.info("""Start encoding"""		)
			logger.info(f"""{len(lowercase__		)} examples to process."""		)

			lowerCAmelCase_    :List[str]			     =  []
			lowerCAmelCase_    :str			     =  0
			lowerCAmelCase_    :Tuple			     =  1_0_0_0_0
			lowerCAmelCase_    :Union[str, Any]			     =  time.time()
			for text in data:
						lowerCAmelCase_    :Any			     =  f"""{bos} {text.strip()} {sep}"""
						lowerCAmelCase_    :List[Any]			     =  tokenizer.encode(lowercase__      ,     add_special_tokens=lowercase__		)
						rslt.append(lowercase__		)

						iter += 1
						if iter % interval == 0:
									lowerCAmelCase_    :Optional[Any]			     =  time.time()
									logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""		)
									lowerCAmelCase_    :int			     =  time.time()
			logger.info("""Finished binarization"""		)
			logger.info(f"""{len(lowercase__		)} examples processed."""		)

			lowerCAmelCase_    :Optional[int]			     =  f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
			lowerCAmelCase_    :Dict			     =  tokenizer.vocab_size
			if vocab_size < (1 << 1_6):
						lowerCAmelCase_    :Any			     =  [np.uintaa(lowercase__		) for d in rslt]
			else:
						lowerCAmelCase_    :str			     =  [np.intaa(lowercase__		) for d in rslt]
			random.shuffle(rslt_		)
			logger.info(f"""Dump to {dp_file}"""		)
			with open(lowercase__      ,     """wb"""		) as handle:
						pickle.dump(rslt_      ,     lowercase__      ,     protocol=pickle.HIGHEST_PROTOCOL		)


if __name__ == "__main__":
				main()












from __future__ import annotations


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     str(_A							)
	return len(_A							) == 9 and set(_A							) == set("123456789"							)


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	for base_num in range(9999     ,	4999     ,	-1							):
		snake_case_		    =     100002 * base_num
		if is_9_pandigital(_A							):
			return candidate

	for base_num in range(333     ,	99     ,	-1							):
		snake_case_		    =     1002003 * base_num
		if is_9_pandigital(_A							):
			return candidate

	return None


if __name__ == "__main__":
		print(f'''{solution() = }''')
