






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A__	:			Any			   = logging.getLogger(__name__)






def   a			(						):






				'''simple docstring'''
				lowercase__    =			argparse.ArgumentParser(
				    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''				)
				parser.add_argument(
				    '''--dataset_name'''		,   type=lowerCamelCase_		,   default='''wikitext'''		,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''		,   )
				parser.add_argument(
				    '''--dataset_config'''		,   type=lowerCamelCase_		,   default='''wikitext-103-raw-v1'''		,   help='''Configuration name of the dataset.'''				)
				parser.add_argument(
				    '''--tokenizer_name_or_path'''		,   type=lowerCamelCase_		,   default='''sayakpaul/unigram-tokenizer-wikitext'''		,   help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''		,   )
				parser.add_argument(
				    '''--shard_size'''		,   type=lowerCamelCase_		,   default=1000		,   help='''Number of entries to go in a single shard.'''		,   )
				parser.add_argument('''--split'''		,   type=lowerCamelCase_		,   default='''train'''		,   choices=['''train''', '''test''', '''validation''']				)
				parser.add_argument(
				    '''--limit'''		,   default=lowerCamelCase_		,   type=lowerCamelCase_		,   help='''Limit the number of shards (used for debugging).'''		,   )
				parser.add_argument(
				    '''--max_length'''		,   type=lowerCamelCase_		,   default=512		,   help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
				    ''' sequence length that is a multiple of 8.'''		,   )
				parser.add_argument(
				    '''--output_dir'''		,   default='''tf-tpu'''		,   type=lowerCamelCase_		,   help='''Output directory where the TFRecord shards will be saved. If the'''
				    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
				    ''' shards will be directly saved to a Google Cloud Storage bucket.'''		,   )

				lowercase__    =			parser.parse_args()
				return args






def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				def fn(lowerCamelCase_				):
								return tokenizer(examples['''text''']				)

				return fn






def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			[]
				for i in range(len(tokenized_data['''input_ids''']				)				):
								lowercase__    =			{
								    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]				)				),
								    '''attention_mask''': tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]				)				),
								}
								lowercase__    =			tf.train.Features(feature=lowerCamelCase_				)
								lowercase__    =			tf.train.Example(features=lowerCamelCase_				)
								lowercase__    =			example.SerializeToString()
								records.append(lowerCamelCase_				)
				return records






def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			datasets.load_dataset(args.dataset_name		,   args.dataset_config		,   split=args.split				)

				if args.limit is not None:
								lowercase__    =			min(len(lowerCamelCase_				)		,   args.limit				)
								lowercase__    =			dataset.select(range(lowerCamelCase_				)				)
								print(F"""Limiting the dataset to {args.limit} entries."""				)

				lowercase__    =			AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir				):
												os.makedirs(args.output_dir				)
								lowercase__    =			os.path.join(args.output_dir		,   args.split				)
								if not os.path.exists(lowerCamelCase_				):
												os.makedirs(lowerCamelCase_				)
				else:
								lowercase__    =			os.path.join(args.output_dir		,   args.split				)

				# Tokenize the whole dataset at once.
				lowercase__    =			tokenize_function(lowerCamelCase_				)
				lowercase__    =			dataset.map(lowerCamelCase_		,   batched=lowerCamelCase_		,   num_proc=4		,   remove_columns=['''text''']				)

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(lowerCamelCase_				):
								# Concatenate all texts.
								lowercase__    =			{k: sum(examples[k]		,   []				) for k in examples.keys()}
								lowercase__    =			len(concatenated_examples[list(examples.keys()				)[0]]				)
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowercase__    =			(total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowercase__    =			{
								    k: [t[i : i + args.max_length] for i in range(0		,   lowerCamelCase_		,   args.max_length				)]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowercase__    =			dataset_tokenized.map(lowerCamelCase_		,   batched=lowerCamelCase_		,   batch_size=1000		,   num_proc=4				)

				lowercase__    =			0
				lowercase__    =			0
				for shard in range(0		,   len(lowerCamelCase_				)		,   args.shard_size				):
								lowercase__    =			grouped_dataset[shard : shard + args.shard_size]
								lowercase__    =			len(dataset_snapshot['''input_ids''']				)
								lowercase__    =			os.path.join(lowerCamelCase_		,   F"""dataset-{shard_count}-{records_containing}.tfrecord"""				)
								lowercase__    =			get_serialized_examples(lowerCamelCase_				)

								with tf.io.TFRecordWriter(lowerCamelCase_				) as out_file:
												for i in range(len(lowerCamelCase_				)				):
																lowercase__    =			serialized_examples[i]
																out_file.write(lowerCamelCase_				)
												print('''Wrote file {} containing {} records'''.format(lowerCamelCase_		,   lowerCamelCase_				)				)

								shard_count += 1
								total_records += records_containing

				with open(F"""split-{args.split}-records-count.txt"""		,   '''w'''				) as f:
								print(F"""Total {args.split} records: {total_records}"""		,   file=lowerCamelCase_				)


if __name__ == "__main__":
							A__	:			List[Any]			   = parse_args()
							main(args)












import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
							import torch_xla.debug.metrics as met
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				def __init__(     self						:    Dict,  *lowerCamelCase						:    Union[str, Any],  lowerCamelCase						:    Union[str, Any]=None,  lowerCamelCase						:    Any=None,  **lowerCamelCase						:    str  ):

								'''simple docstring'''
								super().__init__(*lowerCamelCase,  **lowerCamelCase  )
								lowercase__    =			eval_examples
								lowercase__    =			post_process_function




				def 				lowercase__   (     self						:    int,  lowerCamelCase						:    str=None,  lowerCamelCase						:    Optional[Any]=None,  lowerCamelCase						:    Union[str, Any]=None,  lowerCamelCase						:    str = "eval"  ):

								'''simple docstring'''
								lowercase__    =			self.eval_dataset if eval_dataset is None else eval_dataset
								lowercase__    =			self.get_eval_dataloader(lowerCamelCase  )
								lowercase__    =			self.eval_examples if eval_examples is None else eval_examples

								# Temporarily disable metric computation, we will do it in the loop here.
								lowercase__    =			self.compute_metrics
								lowercase__    =			None
								lowercase__    =			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								lowercase__    =			time.time()
								try:
												lowercase__    =			eval_loop(
												    lowerCamelCase,  description='''Evaluation''',  prediction_loss_only=True if compute_metrics is None else None,  ignore_keys=lowerCamelCase,  metric_key_prefix=lowerCamelCase,  )
								finally:
												lowercase__    =			compute_metrics
								lowercase__    =			self.args.eval_batch_size * self.args.world_size
								if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
												start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
								output.metrics.update(
								    speed_metrics(
								        lowerCamelCase,  lowerCamelCase,  num_samples=output.num_samples,  num_steps=math.ceil(output.num_samples / total_batch_size  ),  )  )
								if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
												# Only the main node write the results by default
												lowercase__    =			self.post_process_function(lowerCamelCase,  lowerCamelCase,  output.predictions  )
												lowercase__    =			self.compute_metrics(lowerCamelCase  )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()  ):
																if not key.startswith(F"""{metric_key_prefix}_"""  ):
																				lowercase__    =			metrics.pop(lowerCamelCase  )
												metrics.update(output.metrics  )
								else:
												lowercase__    =			output.metrics

								if self.args.should_log:
												# Only the main node log the results by default
												self.log(lowerCamelCase  )

								if self.args.tpu_metrics_debug or self.args.debug:
												# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
												xm.master_print(met.metrics_report()  )

								lowercase__    =			self.callback_handler.on_evaluate(self.args,  self.state,  self.control,  lowerCamelCase  )
								return metrics




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    Any,  lowerCamelCase						:    Dict,  lowerCamelCase						:    int=None,  lowerCamelCase						:    str = "test"  ):

								'''simple docstring'''
								lowercase__    =			self.get_test_dataloader(lowerCamelCase  )

								# Temporarily disable metric computation, we will do it in the loop here.
								lowercase__    =			self.compute_metrics
								lowercase__    =			None
								lowercase__    =			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
								lowercase__    =			time.time()
								try:
												lowercase__    =			eval_loop(
												    lowerCamelCase,  description='''Prediction''',  prediction_loss_only=True if compute_metrics is None else None,  ignore_keys=lowerCamelCase,  metric_key_prefix=lowerCamelCase,  )
								finally:
												lowercase__    =			compute_metrics
								lowercase__    =			self.args.eval_batch_size * self.args.world_size
								if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
												start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
								output.metrics.update(
								    speed_metrics(
								        lowerCamelCase,  lowerCamelCase,  num_samples=output.num_samples,  num_steps=math.ceil(output.num_samples / total_batch_size  ),  )  )

								if self.post_process_function is None or self.compute_metrics is None:
												return output

								lowercase__    =			self.post_process_function(lowerCamelCase,  lowerCamelCase,  output.predictions,  '''predict'''  )
								lowercase__    =			self.compute_metrics(lowerCamelCase  )

								# Prefix all keys with metric_key_prefix + '_'
								for key in list(metrics.keys()  ):
												if not key.startswith(F"""{metric_key_prefix}_"""  ):
																lowercase__    =			metrics.pop(lowerCamelCase  )
								metrics.update(output.metrics  )
								return PredictionOutput(predictions=predictions.predictions,  label_ids=predictions.label_ids,  metrics=lowerCamelCase  )




