
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__snake_case								=  logging.getLogger(__name__)






def 							_lowercase							(     )		->		Tuple:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= argparse.ArgumentParser(
					    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'		)
					parser.add_argument(
					    '--dataset_name'	,	type=UpperCamelCase_	,	default='wikitext'	,	help='Name of the training. Explore datasets at: hf.co/datasets.'	,	)
					parser.add_argument(
					    '--dataset_config'	,	type=UpperCamelCase_	,	default='wikitext-103-raw-v1'	,	help='Configuration name of the dataset.'		)
					parser.add_argument(
					    '--tokenizer_name_or_path'	,	type=UpperCamelCase_	,	default='sayakpaul/unigram-tokenizer-wikitext'	,	help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'	,	)
					parser.add_argument(
					    '--shard_size'	,	type=UpperCamelCase_	,	default=1000	,	help='Number of entries to go in a single shard.'	,	)
					parser.add_argument('--split'	,	type=UpperCamelCase_	,	default='train'	,	choices=['train', 'test', 'validation']		)
					parser.add_argument(
					    '--limit'	,	default=UpperCamelCase_	,	type=UpperCamelCase_	,	help='Limit the number of shards (used for debugging).'	,	)
					parser.add_argument(
					    '--max_length'	,	type=UpperCamelCase_	,	default=512	,	help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
					    ' sequence length that is a multiple of 8.'	,	)
					parser.add_argument(
					    '--output_dir'	,	default='tf-tpu'	,	type=UpperCamelCase_	,	help='Output directory where the TFRecord shards will be saved. If the'
					    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
					    ' shards will be directly saved to a Google Cloud Storage bucket.'	,	)

					SCREAMING_SNAKE_CASE__													= parser.parse_args()
					return args






def 							_lowercase							(     UpperCamelCase_		)		->		Any:
					'''simple docstring'''


					def fn(UpperCamelCase_		):
										return tokenizer(examples['text']		)

					return fn






def 							_lowercase							(     UpperCamelCase_		)		->		Dict:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= []
					for i in range(len(tokenized_data['input_ids']		)		):
										SCREAMING_SNAKE_CASE__													= {
										    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]		)		),
										    'attention_mask': tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]		)		),
										}
										SCREAMING_SNAKE_CASE__													= tf.train.Features(feature=UpperCamelCase_		)
										SCREAMING_SNAKE_CASE__													= tf.train.Example(features=UpperCamelCase_		)
										SCREAMING_SNAKE_CASE__													= example.SerializeToString()
										records.append(UpperCamelCase_		)
					return records






def 							_lowercase							(     UpperCamelCase_		)		->		str:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= datasets.load_dataset(args.dataset_name	,	args.dataset_config	,	split=args.split		)

					if args.limit is not None:
										SCREAMING_SNAKE_CASE__													= min(len(UpperCamelCase_		)	,	args.limit		)
										SCREAMING_SNAKE_CASE__													= dataset.select(range(UpperCamelCase_		)		)
										print(F'Limiting the dataset to {args.limit} entries.'		)

					SCREAMING_SNAKE_CASE__													= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir		):
															os.makedirs(args.output_dir		)
										SCREAMING_SNAKE_CASE__													= os.path.join(args.output_dir	,	args.split		)
										if not os.path.exists(UpperCamelCase_		):
															os.makedirs(UpperCamelCase_		)
					else:
										SCREAMING_SNAKE_CASE__													= os.path.join(args.output_dir	,	args.split		)

					# Tokenize the whole dataset at once.
					SCREAMING_SNAKE_CASE__													= tokenize_function(UpperCamelCase_		)
					SCREAMING_SNAKE_CASE__													= dataset.map(UpperCamelCase_	,	batched=UpperCamelCase_	,	num_proc=4	,	remove_columns=['text']		)

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(UpperCamelCase_		):
										# Concatenate all texts.
										SCREAMING_SNAKE_CASE__													= {k: sum(examples[k]	,	[]		) for k in examples.keys()}
										SCREAMING_SNAKE_CASE__													= len(concatenated_examples[list(examples.keys()		)[0]]		)
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										SCREAMING_SNAKE_CASE__													= (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										SCREAMING_SNAKE_CASE__													= {
										    k: [t[i : i + args.max_length] for i in range(0	,	UpperCamelCase_	,	args.max_length		)]
										    for k, t in concatenated_examples.items()
										}
										return result

					SCREAMING_SNAKE_CASE__													= dataset_tokenized.map(UpperCamelCase_	,	batched=UpperCamelCase_	,	batch_size=1000	,	num_proc=4		)

					SCREAMING_SNAKE_CASE__													= 0
					SCREAMING_SNAKE_CASE__													= 0
					for shard in range(0	,	len(UpperCamelCase_		)	,	args.shard_size		):
										SCREAMING_SNAKE_CASE__													= grouped_dataset[shard : shard + args.shard_size]
										SCREAMING_SNAKE_CASE__													= len(dataset_snapshot['input_ids']		)
										SCREAMING_SNAKE_CASE__													= os.path.join(UpperCamelCase_	,	F'dataset-{shard_count}-{records_containing}.tfrecord'		)
										SCREAMING_SNAKE_CASE__													= get_serialized_examples(UpperCamelCase_		)

										with tf.io.TFRecordWriter(UpperCamelCase_		) as out_file:
															for i in range(len(UpperCamelCase_		)		):
																				SCREAMING_SNAKE_CASE__													= serialized_examples[i]
																				out_file.write(UpperCamelCase_		)
															print('Wrote file {} containing {} records'.format(UpperCamelCase_	,	UpperCamelCase_		)		)

										shard_count += 1
										total_records += records_containing

					with open(F'split-{args.split}-records-count.txt'	,	'w'		) as f:
										print(F'Total {args.split} records: {total_records}'	,	file=UpperCamelCase_		)


if __name__ == "__main__":
				__snake_case								=  parse_args()
				main(args)







def 							_lowercase							(     )		->		Optional[int]:
					'''simple docstring'''


					SCREAMING_SNAKE_CASE__													= []
					SCREAMING_SNAKE_CASE__													= 1

					while len(UpperCamelCase_		) < 1e6:
										constant.append(str(UpperCamelCase_		)		)
										i += 1

					SCREAMING_SNAKE_CASE__													= ''.join(UpperCamelCase_		)

					return (
					    int(constant[0]		)
					    * int(constant[9]		)
					    * int(constant[99]		)
					    * int(constant[999]		)
					    * int(constant[9999]		)
					    * int(constant[99999]		)
					    * int(constant[999999]		)
					)


if __name__ == "__main__":
				print(solution())





