


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowerCAmelCase		       =							logging.getLogger(__name__)




def    snake_case_     (  )  ->				Optional[Any]:
       lowercase__:   List[str]  			=   argparse.ArgumentParser(
           description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'     )
       parser.add_argument(
           '--dataset_name'			,							type=snake_case			,							default='wikitext'			,							help='Name of the training. Explore datasets at: hf.co/datasets.'			,							)
       parser.add_argument(
           '--dataset_config'			,							type=snake_case			,							default='wikitext-103-raw-v1'			,							help='Configuration name of the dataset.'     )
       parser.add_argument(
           '--tokenizer_name_or_path'			,							type=snake_case			,							default='sayakpaul/unigram-tokenizer-wikitext'			,							help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'			,							)
       parser.add_argument(
           '--shard_size'			,							type=snake_case			,							default=10_00			,							help='Number of entries to go in a single shard.'			,							)
       parser.add_argument('--split'			,							type=snake_case			,							default='train'			,							choices=['train', 'test', 'validation']     )
       parser.add_argument(
           '--limit'			,							default=snake_case			,							type=snake_case			,							help='Limit the number of shards (used for debugging).'			,							)
       parser.add_argument(
           '--max_length'			,							type=snake_case			,							default=5_12			,							help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
           ' sequence length that is a multiple of 8.'			,							)
       parser.add_argument(
           '--output_dir'			,							default='tf-tpu'			,							type=snake_case			,							help='Output directory where the TFRecord shards will be saved. If the'
           ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
           ' shards will be directly saved to a Google Cloud Storage bucket.'			,							)

       lowercase__:   str  			=   parser.parse_args()
       return args




def    snake_case_     (  snake_case     )  ->				Union[str, Any]:
       def fn(snake_case     ):
              return tokenizer(examples['text']     )

       return fn




def    snake_case_     (  snake_case     )  ->				Optional[int]:
       lowercase__:   Any  			=   []
       for i in range(len(tokenized_data['input_ids']     )     ):
              lowercase__:   Optional[int]  			=   {
                  'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]     )     ),
                  'attention_mask': tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]     )     ),
              }
              lowercase__:   Union[str, Any]  			=   tf.train.Features(feature=snake_case     )
              lowercase__:   Optional[int]  			=   tf.train.Example(features=snake_case     )
              lowercase__:   str  			=   example.SerializeToString()
              records.append(snake_case     )
       return records




def    snake_case_     (  snake_case     )  ->				Optional[int]:
       lowercase__:   Optional[Any]  			=   datasets.load_dataset(args.dataset_name			,							args.dataset_config			,							split=args.split     )

       if args.limit is not None:
              lowercase__:   Any  			=   min(len(snake_case     )			,							args.limit     )
              lowercase__:   Optional[int]  			=   dataset.select(range(snake_case     )     )
              print(f'Limiting the dataset to {args.limit} entries.'     )

       lowercase__:   Dict  			=   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir     ):
                     os.makedirs(args.output_dir     )
              lowercase__:   Optional[Any]  			=   os.path.join(args.output_dir			,							args.split     )
              if not os.path.exists(snake_case     ):
                     os.makedirs(snake_case     )
       else:
              lowercase__:   Tuple  			=   os.path.join(args.output_dir			,							args.split     )

       # Tokenize the whole dataset at once.
       lowercase__:   List[str]  			=   tokenize_function(snake_case     )
       lowercase__:   List[str]  			=   dataset.map(snake_case			,							batched=snake_case			,							num_proc=4			,							remove_columns=['text']     )

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(snake_case     ):
              # Concatenate all texts.
              lowercase__:   Any  			=   {k: sum(examples[k]			,							[]     ) for k in examples.keys()}
              lowercase__:   int  			=   len(concatenated_examples[list(examples.keys()     )[0]]     )
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              lowercase__:   Tuple  			=   (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              lowercase__:   Union[str, Any]  			=   {
                  k: [t[i : i + args.max_length] for i in range(0			,							snake_case			,							args.max_length     )]
                  for k, t in concatenated_examples.items()
              }
              return result

       lowercase__:   Any  			=   dataset_tokenized.map(snake_case			,							batched=snake_case			,							batch_size=10_00			,							num_proc=4     )

       lowercase__:   List[Any]  			=   0
       lowercase__:   Dict  			=   0
       for shard in range(0			,							len(snake_case     )			,							args.shard_size     ):
              lowercase__:   Optional[Any]  			=   grouped_dataset[shard : shard + args.shard_size]
              lowercase__:   Union[str, Any]  			=   len(dataset_snapshot['input_ids']     )
              lowercase__:   Optional[int]  			=   os.path.join(snake_case			,							f'dataset-{shard_count}-{records_containing}.tfrecord'     )
              lowercase__:   Any  			=   get_serialized_examples(snake_case     )

              with tf.io.TFRecordWriter(snake_case     ) as out_file:
                     for i in range(len(snake_case     )     ):
                            lowercase__:   List[Any]  			=   serialized_examples[i]
                            out_file.write(snake_case     )
                     print('Wrote file {} containing {} records'.format(snake_case			,							snake_case     )     )

              shard_count += 1
              total_records += records_containing

       with open(f'split-{args.split}-records-count.txt'			,							'w'     ) as f:
              print(f'Total {args.split} records: {total_records}'			,							file=snake_case     )


if __name__ == "__main__":
 __lowerCAmelCase		       =							parse_args()
 main(args)






def    snake_case_     (  snake_case			,							snake_case     )  ->				bool:
       return numa ^ numa < 0


if __name__ == "__main__":
 import doctest

 doctest.testmod()


