




'''simple docstring'''


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase       			= logging.getLogger(__name__)
def  SCREAMING_SNAKE_CASE(					)       ->      Dict:
     A:						Optional[Any]						=						argparse.ArgumentParser(
         description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''     )
     parser.add_argument(
         '''--dataset_name'''						,							type=__lowercase						,							default='''wikitext'''						,							help='''Name of the training. Explore datasets at: hf.co/datasets.'''						,							)
     parser.add_argument(
         '''--dataset_config'''						,							type=__lowercase						,							default='''wikitext-103-raw-v1'''						,							help='''Configuration name of the dataset.'''     )
     parser.add_argument(
         '''--tokenizer_name_or_path'''						,							type=__lowercase						,							default='''sayakpaul/unigram-tokenizer-wikitext'''						,							help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''						,							)
     parser.add_argument(
         '''--shard_size'''						,							type=__lowercase						,							default=1_0_0_0						,							help='''Number of entries to go in a single shard.'''						,							)
     parser.add_argument('''--split'''						,							type=__lowercase						,							default='''train'''						,							choices=['''train''', '''test''', '''validation''']     )
     parser.add_argument(
         '''--limit'''						,							default=__lowercase						,							type=__lowercase						,							help='''Limit the number of shards (used for debugging).'''						,							)
     parser.add_argument(
         '''--max_length'''						,							type=__lowercase						,							default=5_1_2						,							help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
         ''' sequence length that is a multiple of 8.'''						,							)
     parser.add_argument(
         '''--output_dir'''						,							default='''tf-tpu'''						,							type=__lowercase						,							help='''Output directory where the TFRecord shards will be saved. If the'''
         ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
         ''' shards will be directly saved to a Google Cloud Storage bucket.'''						,							)

     A:						Any						=						parser.parse_args()
     return args
def  SCREAMING_SNAKE_CASE(					__lowercase     )       ->      List[str]:



     def fn(__lowercase     ):
          return tokenizer(examples['''text''']     )

     return fn
def  SCREAMING_SNAKE_CASE(					__lowercase     )       ->      str:
     A:						List[str]						=						[]
     for i in range(len(tokenized_data['''input_ids''']     )     ):
          A:						List[str]						=						{
              '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]     )     ),
              '''attention_mask''': tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]     )     ),
          }
          A:						List[str]						=						tf.train.Features(feature=__lowercase     )
          A:						Any						=						tf.train.Example(features=__lowercase     )
          A:						List[str]						=						example.SerializeToString()
          records.append(__lowercase     )
     return records
def  SCREAMING_SNAKE_CASE(					__lowercase     )       ->      Optional[int]:
     A:						str						=						datasets.load_dataset(args.dataset_name						,							args.dataset_config						,							split=args.split     )

     if args.limit is not None:
          A:						Optional[int]						=						min(len(__lowercase     )						,							args.limit     )
          A:						Union[str, Any]						=						dataset.select(range(__lowercase     )     )
          print(F"""Limiting the dataset to {args.limit} entries."""     )

     A:						Optional[int]						=						AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir     ):
               os.makedirs(args.output_dir     )
          A:						Optional[Any]						=						os.path.join(args.output_dir						,							args.split     )
          if not os.path.exists(__lowercase     ):
               os.makedirs(__lowercase     )
     else:
          A:						int						=						os.path.join(args.output_dir						,							args.split     )

     # Tokenize the whole dataset at once.
     A:						Optional[int]						=						tokenize_function(__lowercase     )
     A:						Optional[Any]						=						dataset.map(__lowercase						,							batched=__lowercase						,							num_proc=4						,							remove_columns=['''text''']     )

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(__lowercase     ):
          # Concatenate all texts.
          A:						Any						=						{k: sum(examples[k]						,							[]     ) for k in examples.keys()}
          A:						List[Any]						=						len(concatenated_examples[list(examples.keys()     )[0]]     )
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          A:						List[Any]						=						(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          A:						Tuple						=						{
              k: [t[i : i + args.max_length] for i in range(0						,							__lowercase						,							args.max_length     )]
              for k, t in concatenated_examples.items()
          }
          return result

     A:						List[str]						=						dataset_tokenized.map(__lowercase						,							batched=__lowercase						,							batch_size=1_0_0_0						,							num_proc=4     )

     A:						Tuple						=						0
     A:						Any						=						0
     for shard in range(0						,							len(__lowercase     )						,							args.shard_size     ):
          A:						Optional[Any]						=						grouped_dataset[shard : shard + args.shard_size]
          A:						Any						=						len(dataset_snapshot['''input_ids''']     )
          A:						Tuple						=						os.path.join(__lowercase						,							F"""dataset-{shard_count}-{records_containing}.tfrecord"""     )
          A:						Optional[int]						=						get_serialized_examples(__lowercase     )

          with tf.io.TFRecordWriter(__lowercase     ) as out_file:
               for i in range(len(__lowercase     )     ):
                    A:						List[str]						=						serialized_examples[i]
                    out_file.write(__lowercase     )
               print('''Wrote file {} containing {} records'''.format(__lowercase						,							__lowercase     )     )

          shard_count += 1
          total_records += records_containing

     with open(F"""split-{args.split}-records-count.txt"""						,							'''w'''     ) as f:
          print(F"""Total {args.split} records: {total_records}"""						,							file=__lowercase     )


if __name__ == "__main__":
     UpperCamelCase       			= parse_args()
     main(args)







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__a  :     Optional[int]       =  logging.get_logger(__name__)

__a  :     Tuple       =  {
    """facebook/data2vec-text-base""": """https://huggingface.co/data2vec/resolve/main/config.json""",
}




class    _UpperCamelCase      (						_UpperCAmelCase ):


			"""simple docstring"""






			__a  : Dict			 =						'''data2vec-text'''





			def __init__(   self      ,   lowerCAmelCase__=3_05_22      ,   lowerCAmelCase__=7_68      ,   lowerCAmelCase__=12      ,   lowerCAmelCase__=12      ,   lowerCAmelCase__=30_72      ,   lowerCAmelCase__="gelu"      ,   lowerCAmelCase__=0.1      ,   lowerCAmelCase__=0.1      ,   lowerCAmelCase__=5_12      ,   lowerCAmelCase__=2      ,   lowerCAmelCase__=0.02      ,   lowerCAmelCase__=1E-12      ,   lowerCAmelCase__=1      ,   lowerCAmelCase__=0      ,   lowerCAmelCase__=2      ,   lowerCAmelCase__="absolute"      ,   lowerCAmelCase__=True      ,   lowerCAmelCase__=None      ,   **lowerCAmelCase__      ,   )     ->					Any:

							'''simple docstring'''


							super().__init__(pad_token_id=lowerCAmelCase__      ,   bos_token_id=lowerCAmelCase__      ,   eos_token_id=lowerCAmelCase__      ,   **lowerCAmelCase__						)

							__lowercase 			=							vocab_size
							__lowercase 			=							hidden_size
							__lowercase 			=							num_hidden_layers
							__lowercase 			=							num_attention_heads
							__lowercase 			=							hidden_act
							__lowercase 			=							intermediate_size
							__lowercase 			=							hidden_dropout_prob
							__lowercase 			=							attention_probs_dropout_prob
							__lowercase 			=							max_position_embeddings
							__lowercase 			=							type_vocab_size
							__lowercase 			=							initializer_range
							__lowercase 			=							layer_norm_eps
							__lowercase 			=							position_embedding_type
							__lowercase 			=							use_cache
							__lowercase 			=							classifier_dropout




class    _UpperCamelCase      (						_UpperCAmelCase ):


			"""simple docstring"""






			@property
			def        _SCREAMING_SNAKE_CASE			(   self						)     ->					Mapping[str, Mapping[int, str]]:

							'''simple docstring'''


							if self.task == "multiple-choice":
											__lowercase 			=							{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
							else:
											__lowercase 			=							{0: '''batch''', 1: '''sequence'''}
							return OrderedDict(
							    [
							        ('''input_ids''', dynamic_axis),
							        ('''attention_mask''', dynamic_axis),
							    ]						)