






import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser
def 					snake_case_						(   lowerCAmelCase_				:  Optional[int]					):
							__lowercase					:							int			=   {}
							__lowercase					:							Optional[int]			=   tokenizer(example["""content"""]				,				truncation=lowerCAmelCase_					)["""input_ids"""]
							__lowercase					:							Tuple			=   len(example["""content"""]					) / len(output["""input_ids"""]					)
							return output


lowerCamelCase       :       Optional[int]        =				HfArgumentParser(PretokenizationArguments)
lowerCamelCase       :       int        =				parser.parse_args()
if args.num_workers is None:
	lowerCamelCase       :       List[Any]        =				multiprocessing.cpu_count()
lowerCamelCase       :       Optional[Any]        =				AutoTokenizer.from_pretrained(args.tokenizer_dir)

lowerCamelCase       :       int        =				time.time()
lowerCamelCase       :       List[str]        =				load_dataset(args.dataset_name, split='''train''')
print(f'''Dataset loaded in {time.time()-t_start:.2f}s''')

lowerCamelCase       :       List[Any]        =				time.time()
lowerCamelCase       :       str        =				ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        '''repo_name''',
        '''path''',
        '''copies''',
        '''size''',
        '''content''',
        '''license''',
        '''hash''',
        '''line_mean''',
        '''line_max''',
        '''alpha_frac''',
        '''autogenerated''',
    ],
)
print(f'''Dataset tokenized in {time.time()-t_start:.2f}s''')

lowerCamelCase       :       Optional[Any]        =				time.time()
ds.push_to_hub(args.tokenized_data_repo)



print(f'''Data pushed to the hub in {time.time()-t_start:.2f}s''')







from maths.prime_check import is_prime
def 					snake_case_						(   lowerCAmelCase_				:  int					):
							if not isinstance(lowerCAmelCase_				,				lowerCAmelCase_					):
														__lowercase					:							Dict			=   F"Input value of [number={number}] must be an integer"
														raise TypeError(lowerCAmelCase_					)
							if is_prime(lowerCAmelCase_					) and is_prime(number + 2					):
														return number + 2
							else:
														return -1


if __name__ == "__main__":
	import doctest

	doctest.testmod()