"""simple docstring"""





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
_UpperCamelCase   :						Any      =    logging.getLogger(__name__)





def 				a_     (      ):





       '''simple docstring'''

       lowercase__			:		Union[str, Any]       =	argparse.ArgumentParser(
           description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'     )
       parser.add_argument('--file_path'   , type=_lowerCAmelCase   , default='data/dump.txt'   , help='The path to the data.'     )
       parser.add_argument('--tokenizer_type'   , type=_lowerCAmelCase   , default='bert'   , choices=['bert', 'roberta', 'gpt2']     )
       parser.add_argument('--tokenizer_name'   , type=_lowerCAmelCase   , default='bert-base-uncased'   , help='The tokenizer to use.'     )
       parser.add_argument('--dump_file'   , type=_lowerCAmelCase   , default='data/dump'   , help='The dump file prefix.'     )
       lowercase__			:		Any       =	parser.parse_args()

       logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""     )
       if args.tokenizer_type == "bert":
              lowercase__			:		Tuple       =	BertTokenizer.from_pretrained(args.tokenizer_name     )
              lowercase__			:		Optional[int]       =	tokenizer.special_tokens_map['cls_token']  # `[CLS]`
              lowercase__			:		List[str]       =	tokenizer.special_tokens_map['sep_token']  # `[SEP]`
       elif args.tokenizer_type == "roberta":
              lowercase__			:		Optional[int]       =	RobertaTokenizer.from_pretrained(args.tokenizer_name     )
              lowercase__			:		Dict       =	tokenizer.special_tokens_map['cls_token']  # `<s>`
              lowercase__			:		Any       =	tokenizer.special_tokens_map['sep_token']  # `</s>`
       elif args.tokenizer_type == "gpt2":
              lowercase__			:		List[str]       =	GPTaTokenizer.from_pretrained(args.tokenizer_name     )
              lowercase__			:		Any       =	tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
              lowercase__			:		Tuple       =	tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

       logger.info(f"""Loading text from {args.file_path}"""     )
       with open(args.file_path   , 'r'   , encoding='utf8'     ) as fp:
              lowercase__			:		int       =	fp.readlines()

       logger.info('Start encoding'     )
       logger.info(f"""{len(_lowerCAmelCase     )} examples to process."""     )

       lowercase__			:		Optional[Any]       =	[]
       lowercase__			:		Optional[int]       =	0
       lowercase__			:		List[Any]       =	1_0000
       lowercase__			:		int       =	time.time()
       for text in data:
              lowercase__			:		Any       =	f"""{bos} {text.strip()} {sep}"""
              lowercase__			:		Optional[int]       =	tokenizer.encode(_lowerCAmelCase   , add_special_tokens=_lowerCAmelCase     )
              rslt.append(_lowerCAmelCase     )

              iter += 1
              if iter % interval == 0:
                     lowercase__			:		List[str]       =	time.time()
                     logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""     )
                     lowercase__			:		List[Any]       =	time.time()
       logger.info('Finished binarization'     )
       logger.info(f"""{len(_lowerCAmelCase     )} examples processed."""     )

       lowercase__			:		Tuple       =	f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
       lowercase__			:		Any       =	tokenizer.vocab_size
       if vocab_size < (1 << 16):
              lowercase__			:		Tuple       =	[np.uintaa(_lowerCAmelCase     ) for d in rslt]
       else:
              lowercase__			:		Dict       =	[np.intaa(_lowerCAmelCase     ) for d in rslt]
       random.shuffle(rslt_     )
       logger.info(f"""Dump to {dp_file}"""     )
       with open(_lowerCAmelCase   , 'wb'     ) as handle:
              pickle.dump(rslt_   , _lowerCAmelCase   , protocol=pickle.HIGHEST_PROTOCOL     )


if __name__ == "__main__":
       main()







_UpperCAmelCase		:  Tuple	=   {str(digit): digit**5 for digit in range(10)}



def 					UpperCAmelCase__     (   lowerCamelCase      ):
  return sum(DIGITS_FIFTH_POWER[digit] for digit in str(lowerCamelCase      )      )



def 					UpperCAmelCase__     (   ):
  return sum(
      number
      for number in range(1000,	1000000      )
      if number == digits_fifth_powers_sum(lowerCamelCase      )      )


if __name__ == "__main__":
     print(solution())



