



'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A_							: Union[str, Any]						 =	logging.getLogger(__name__)




def     snake_case_					(     )->		Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase  :						List[Any]			     = argparse.ArgumentParser(
           description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""		)
       parser.add_argument(
           """--dataset_name"""       ,   type=lowerCAmelCase_       ,   default="""wikitext"""       ,   help="""Name of the training. Explore datasets at: hf.co/datasets."""       ,   )
       parser.add_argument(
           """--dataset_config"""       ,   type=lowerCAmelCase_       ,   default="""wikitext-103-raw-v1"""       ,   help="""Configuration name of the dataset."""		)
       parser.add_argument(
           """--tokenizer_name_or_path"""       ,   type=lowerCAmelCase_       ,   default="""sayakpaul/unigram-tokenizer-wikitext"""       ,   help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""       ,   )
       parser.add_argument(
           """--shard_size"""       ,   type=lowerCAmelCase_       ,   default=1000       ,   help="""Number of entries to go in a single shard."""       ,   )
       parser.add_argument("""--split"""       ,   type=lowerCAmelCase_       ,   default="""train"""       ,   choices=["""train""", """test""", """validation"""]		)
       parser.add_argument(
           """--limit"""       ,   default=lowerCAmelCase_       ,   type=lowerCAmelCase_       ,   help="""Limit the number of shards (used for debugging)."""       ,   )
       parser.add_argument(
           """--max_length"""       ,   type=lowerCAmelCase_       ,   default=512       ,   help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
           """ sequence length that is a multiple of 8."""       ,   )
       parser.add_argument(
           """--output_dir"""       ,   default="""tf-tpu"""       ,   type=lowerCAmelCase_       ,   help="""Output directory where the TFRecord shards will be saved. If the"""
           """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
           """ shards will be directly saved to a Google Cloud Storage bucket."""       ,   )

       _UpperCAmelCase  :						Tuple			     = parser.parse_args()
       return args




def     snake_case_					(     lowerCAmelCase_		)->		Optional[Any]:





       '''simple docstring'''


       def fn(lowerCAmelCase_		):
              return tokenizer(examples["""text"""]		)

       return fn




def     snake_case_					(     lowerCAmelCase_		)->		str:





       '''simple docstring'''


       _UpperCAmelCase  :						List[str]			     = []
       for i in range(len(tokenized_data["""input_ids"""]		)		):
              _UpperCAmelCase  :						List[str]			     = {
                  """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]		)		),
                  """attention_mask""": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]		)		),
              }
              _UpperCAmelCase  :						Optional[Any]			     = tf.train.Features(feature=lowerCAmelCase_		)
              _UpperCAmelCase  :						Any			     = tf.train.Example(features=lowerCAmelCase_		)
              _UpperCAmelCase  :						List[str]			     = example.SerializeToString()
              records.append(lowerCAmelCase_		)
       return records




def     snake_case_					(     lowerCAmelCase_		)->		Dict:





       '''simple docstring'''


       _UpperCAmelCase  :						Any			     = datasets.load_dataset(args.dataset_name       ,   args.dataset_config       ,   split=args.split		)

       if args.limit is not None:
              _UpperCAmelCase  :						Union[str, Any]			     = min(len(lowerCAmelCase_		)       ,   args.limit		)
              _UpperCAmelCase  :						int			     = dataset.select(range(lowerCAmelCase_		)		)
              print(F'''Limiting the dataset to {args.limit} entries.'''		)

       _UpperCAmelCase  :						str			     = AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir		):
                     os.makedirs(args.output_dir		)
              _UpperCAmelCase  :						Optional[int]			     = os.path.join(args.output_dir       ,   args.split		)
              if not os.path.exists(lowerCAmelCase_		):
                     os.makedirs(lowerCAmelCase_		)
       else:
              _UpperCAmelCase  :						Any			     = os.path.join(args.output_dir       ,   args.split		)

       # Tokenize the whole dataset at once.
       _UpperCAmelCase  :						List[Any]			     = tokenize_function(lowerCAmelCase_		)
       _UpperCAmelCase  :						Tuple			     = dataset.map(lowerCAmelCase_       ,   batched=lowerCAmelCase_       ,   num_proc=4       ,   remove_columns=["""text"""]		)

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(lowerCAmelCase_		):
              # Concatenate all texts.
              _UpperCAmelCase  :						Dict			     = {k: sum(examples[k]       ,   []		) for k in examples.keys()}
              _UpperCAmelCase  :						Any			     = len(concatenated_examples[list(examples.keys()		)[0]]		)
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              _UpperCAmelCase  :						Tuple			     = (total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              _UpperCAmelCase  :						Any			     = {
                  k: [t[i : i + args.max_length] for i in range(0       ,   lowerCAmelCase_       ,   args.max_length		)]
                  for k, t in concatenated_examples.items()
              }
              return result

       _UpperCAmelCase  :						Tuple			     = dataset_tokenized.map(lowerCAmelCase_       ,   batched=lowerCAmelCase_       ,   batch_size=1000       ,   num_proc=4		)

       _UpperCAmelCase  :						Union[str, Any]			     = 0
       _UpperCAmelCase  :						Optional[int]			     = 0
       for shard in range(0       ,   len(lowerCAmelCase_		)       ,   args.shard_size		):
              _UpperCAmelCase  :						int			     = grouped_dataset[shard : shard + args.shard_size]
              _UpperCAmelCase  :						Optional[int]			     = len(dataset_snapshot["""input_ids"""]		)
              _UpperCAmelCase  :						Dict			     = os.path.join(lowerCAmelCase_       ,   F'''dataset-{shard_count}-{records_containing}.tfrecord'''		)
              _UpperCAmelCase  :						int			     = get_serialized_examples(lowerCAmelCase_		)

              with tf.io.TFRecordWriter(lowerCAmelCase_		) as out_file:
                     for i in range(len(lowerCAmelCase_		)		):
                            _UpperCAmelCase  :						Dict			     = serialized_examples[i]
                            out_file.write(lowerCAmelCase_		)
                     print("""Wrote file {} containing {} records""".format(lowerCAmelCase_       ,   lowerCAmelCase_		)		)

              shard_count += 1
              total_records += records_containing

       with open(F'''split-{args.split}-records-count.txt'''       ,   """w"""		) as f:
              print(F'''Total {args.split} records: {total_records}'''       ,   file=lowerCAmelCase_		)


if __name__ == "__main__":
   A_							: Optional[Any]						 =	parse_args()
   main(args)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A_							: Optional[int]						 =	{"""configuration_ibert""": ["""IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """IBertConfig""", """IBertOnnxConfig"""]}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: str						 =	[
       """IBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """IBertForMaskedLM""",
       """IBertForMultipleChoice""",
       """IBertForQuestionAnswering""",
       """IBertForSequenceClassification""",
       """IBertForTokenClassification""",
       """IBertModel""",
       """IBertPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_ibert import (
          IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          IBertForMaskedLM,
          IBertForMultipleChoice,
          IBertForQuestionAnswering,
          IBertForSequenceClassification,
          IBertForTokenClassification,
          IBertModel,
          IBertPreTrainedModel,
      )

else:
   import sys

   A_							: str						 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



