






"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_UpperCamelCase		:  Dict						=		logging.getLogger(__name__)
def 						_SCREAMING_SNAKE_CASE (    ):





  '''simple docstring'''





  lowercase							    =       argparse.ArgumentParser(
      description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'    )
  parser.add_argument(
      '--dataset_name'		,					type=__snake_case		,					default='wikitext'		,					help='Name of the training. Explore datasets at: hf.co/datasets.'		,					)
  parser.add_argument(
      '--dataset_config'		,					type=__snake_case		,					default='wikitext-103-raw-v1'		,					help='Configuration name of the dataset.'    )
  parser.add_argument(
      '--tokenizer_name_or_path'		,					type=__snake_case		,					default='sayakpaul/unigram-tokenizer-wikitext'		,					help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'		,					)
  parser.add_argument(
      '--shard_size'		,					type=__snake_case		,					default=10_00		,					help='Number of entries to go in a single shard.'		,					)
  parser.add_argument('--split'		,					type=__snake_case		,					default='train'		,					choices=['train', 'test', 'validation']    )
  parser.add_argument(
      '--limit'		,					default=__snake_case		,					type=__snake_case		,					help='Limit the number of shards (used for debugging).'		,					)
  parser.add_argument(
      '--max_length'		,					type=__snake_case		,					default=5_12		,					help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
      ' sequence length that is a multiple of 8.'		,					)
  parser.add_argument(
      '--output_dir'		,					default='tf-tpu'		,					type=__snake_case		,					help='Output directory where the TFRecord shards will be saved. If the'
      ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
      ' shards will be directly saved to a Google Cloud Storage bucket.'		,					)

  lowercase							    =       parser.parse_args()
  return args
def 						_SCREAMING_SNAKE_CASE (    __snake_case			:  Optional[int]    ):





  '''simple docstring'''





  def fn(__snake_case			:  Tuple    ):
    return tokenizer(examples['text']    )

  return fn
def 						_SCREAMING_SNAKE_CASE (    __snake_case			:  Any    ):





  '''simple docstring'''





  lowercase							    =       []
  for i in range(len(tokenized_data['input_ids']    )    ):
    lowercase							    =       {
        'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]    )    ),
        'attention_mask': tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]    )    ),
    }
    lowercase							    =       tf.train.Features(feature=__snake_case    )
    lowercase							    =       tf.train.Example(features=__snake_case    )
    lowercase							    =       example.SerializeToString()
    records.append(__snake_case    )
  return records
def 						_SCREAMING_SNAKE_CASE (    __snake_case			:  Union[str, Any]    ):





  '''simple docstring'''





  lowercase							    =       datasets.load_dataset(args.dataset_name		,					args.dataset_config		,					split=args.split    )

  if args.limit is not None:
    lowercase							    =       min(len(__snake_case    )		,					args.limit    )
    lowercase							    =       dataset.select(range(__snake_case    )    )
    print(f'Limiting the dataset to {args.limit} entries.'    )

  lowercase							    =       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir    ):
      os.makedirs(args.output_dir    )
    lowercase							    =       os.path.join(args.output_dir		,					args.split    )
    if not os.path.exists(__snake_case    ):
      os.makedirs(__snake_case    )
  else:
    lowercase							    =       os.path.join(args.output_dir		,					args.split    )

  # Tokenize the whole dataset at once.
  lowercase							    =       tokenize_function(__snake_case    )
  lowercase							    =       dataset.map(__snake_case		,					batched=__snake_case		,					num_proc=4		,					remove_columns=['text']    )

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(__snake_case			:  str    ):
    # Concatenate all texts.
    lowercase							    =       {k: sum(examples[k]		,					[]    ) for k in examples.keys()}
    lowercase							    =       len(concatenated_examples[list(examples.keys()    )[0]]    )
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    lowercase							    =       (total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    lowercase							    =       {
        k: [t[i : i + args.max_length] for i in range(0		,					__snake_case		,					args.max_length    )]
        for k, t in concatenated_examples.items()
    }
    return result

  lowercase							    =       dataset_tokenized.map(__snake_case		,					batched=__snake_case		,					batch_size=10_00		,					num_proc=4    )

  lowercase							    =       0
  lowercase							    =       0
  for shard in range(0		,					len(__snake_case    )		,					args.shard_size    ):
    lowercase							    =       grouped_dataset[shard : shard + args.shard_size]
    lowercase							    =       len(dataset_snapshot['input_ids']    )
    lowercase							    =       os.path.join(__snake_case		,					f'dataset-{shard_count}-{records_containing}.tfrecord'    )
    lowercase							    =       get_serialized_examples(__snake_case    )

    with tf.io.TFRecordWriter(__snake_case    ) as out_file:
      for i in range(len(__snake_case    )    ):
        lowercase							    =       serialized_examples[i]
        out_file.write(__snake_case    )
      print('Wrote file {} containing {} records'.format(__snake_case		,					__snake_case    )    )

    shard_count += 1
    total_records += records_containing

  with open(f'split-{args.split}-records-count.txt'		,					'w'    ) as f:
    print(f'Total {args.split} records: {total_records}'		,					file=__snake_case    )


if __name__ == "__main__":
    _UpperCamelCase		:  Optional[Any]						=		parse_args()
    main(args)










"""simple docstring"""



# Author: OMKAR PATHAK, Nwachukwu Chidiebere

# Use a Python dictionary to construct the graph.
from __future__ import annotations

from pprint import pformat
from typing import Generic, TypeVar

_UpperCamelCase		:  Optional[Any]						=		TypeVar('T')

class        a     (   Generic[T]	):







    def __init__(			self							,		_lowerCamelCase = True       ):
      lowercase							    =       {}  # dictionary of lists
      lowercase							    =       directed







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase       ):

      if not self.directed:  # For undirected graphs
        # if both source vertex and destination vertex are both present in the
        # adjacency list, add destination vertex to source vertex list of adjacent
        # vertices and add source vertex to destination vertex list of adjacent
        # vertices.
        if source_vertex in self.adj_list and destination_vertex in self.adj_list:
          self.adj_list[source_vertex].append(_lowerCamelCase       )
          self.adj_list[destination_vertex].append(_lowerCamelCase       )
        # if only source vertex is present in adjacency list, add destination vertex
        # to source vertex list of adjacent vertices, then create a new vertex with
        # destination vertex as key and assign a list containing the source vertex
        # as it's first adjacent vertex.
        elif source_vertex in self.adj_list:
          self.adj_list[source_vertex].append(_lowerCamelCase       )
          lowercase							    =       [source_vertex]
        # if only destination vertex is present in adjacency list, add source vertex
        # to destination vertex list of adjacent vertices, then create a new vertex
        # with source vertex as key and assign a list containing the source vertex
        # as it's first adjacent vertex.
        elif destination_vertex in self.adj_list:
          self.adj_list[destination_vertex].append(_lowerCamelCase       )
          lowercase							    =       [destination_vertex]
        # if both source vertex and destination vertex are not present in adjacency
        # list, create a new vertex with source vertex as key and assign a list
        # containing the destination vertex as it's first adjacent vertex also
        # create a new vertex with destination vertex as key and assign a list
        # containing the source vertex as it's first adjacent vertex.
        else:
          lowercase							    =       [destination_vertex]
          lowercase							    =       [source_vertex]
      else:  # For directed graphs
        # if both source vertex and destination vertex are present in adjacency
        # list, add destination vertex to source vertex list of adjacent vertices.
        if source_vertex in self.adj_list and destination_vertex in self.adj_list:
          self.adj_list[source_vertex].append(_lowerCamelCase       )
        # if only source vertex is present in adjacency list, add destination
        # vertex to source vertex list of adjacent vertices and create a new vertex
        # with destination vertex as key, which has no adjacent vertex
        elif source_vertex in self.adj_list:
          self.adj_list[source_vertex].append(_lowerCamelCase       )
          lowercase							    =       []
        # if only destination vertex is present in adjacency list, create a new
        # vertex with source vertex as key and assign a list containing destination
        # vertex as first adjacent vertex
        elif destination_vertex in self.adj_list:
          lowercase							    =       [destination_vertex]
        # if both source vertex and destination vertex are not present in adjacency
        # list, create a new vertex with source vertex as key and a list containing
        # destination vertex as it's first adjacent vertex. Then create a new vertex
        # with destination vertex as key, which has no adjacent vertex
        else:
          lowercase							    =       [destination_vertex]
          lowercase							    =       []

      return self







    def __repr__(			self       ):
      return pformat(self.adj_list       )


