
"""simple docstring"""




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_A			:       Any   		=       logging.getLogger(__name__)


def   __magic_name__	(      ) ->				str:
       lowercase    :				Optional[int]       		=							argparse.ArgumentParser(
           description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."  )
       parser.add_argument(
           "--dataset_name"     , type=__snake_case     , default="wikitext"     , help="Name of the training. Explore datasets at: hf.co/datasets."     , )
       parser.add_argument(
           "--dataset_config"     , type=__snake_case     , default="wikitext-103-raw-v1"     , help="Configuration name of the dataset."  )
       parser.add_argument(
           "--tokenizer_name_or_path"     , type=__snake_case     , default="sayakpaul/unigram-tokenizer-wikitext"     , help="Tokenizer identifier. Can be a local filepath or a Hub identifier."     , )
       parser.add_argument(
           "--shard_size"     , type=__snake_case     , default=1000     , help="Number of entries to go in a single shard."     , )
       parser.add_argument("--split"     , type=__snake_case     , default="train"     , choices=["train", "test", "validation"]  )
       parser.add_argument(
           "--limit"     , default=__snake_case     , type=__snake_case     , help="Limit the number of shards (used for debugging)."     , )
       parser.add_argument(
           "--max_length"     , type=__snake_case     , default=512     , help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
           " sequence length that is a multiple of 8."     , )
       parser.add_argument(
           "--output_dir"     , default="tf-tpu"     , type=__snake_case     , help="Output directory where the TFRecord shards will be saved. If the"
           " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
           " shards will be directly saved to a Google Cloud Storage bucket."     , )

       lowercase    :				str       		=							parser.parse_args()
       return args


def   __magic_name__	(      __snake_case				:       Optional[Any]  ) ->				Optional[int]:
       def fn(__snake_case				:       Optional[int]  ):
              return tokenizer(examples["text"]  )

       return fn


def   __magic_name__	(      __snake_case				:       List[Any]  ) ->				List[str]:
       lowercase    :				List[str]       		=							[]
       for i in range(len(tokenized_data["input_ids"]  )  ):
              lowercase    :				Union[str, Any]       		=							{
                  "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]  )  ),
                  "attention_mask": tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]  )  ),
              }
              lowercase    :				str       		=							tf.train.Features(feature=__snake_case  )
              lowercase    :				List[Any]       		=							tf.train.Example(features=__snake_case  )
              lowercase    :				Dict       		=							example.SerializeToString()
              records.append(__snake_case  )
       return records


def   __magic_name__	(      __snake_case				:       str  ) ->				Tuple:
       lowercase    :				Optional[int]       		=							datasets.load_dataset(args.dataset_name     , args.dataset_config     , split=args.split  )

       if args.limit is not None:
              lowercase    :				List[str]       		=							min(len(__snake_case  )     , args.limit  )
              lowercase    :				str       		=							dataset.select(range(__snake_case  )  )
              print(f"""Limiting the dataset to {args.limit} entries."""  )

       lowercase    :				str       		=							AutoTokenizer.from_pretrained(args.tokenizer_name_or_path  )

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir  ):
                     os.makedirs(args.output_dir  )
              lowercase    :				str       		=							os.path.join(args.output_dir     , args.split  )
              if not os.path.exists(__snake_case  ):
                     os.makedirs(__snake_case  )
       else:
              lowercase    :				Union[str, Any]       		=							os.path.join(args.output_dir     , args.split  )

       # Tokenize the whole dataset at once.
       lowercase    :				str       		=							tokenize_function(__snake_case  )
       lowercase    :				Union[str, Any]       		=							dataset.map(__snake_case     , batched=__snake_case     , num_proc=4     , remove_columns=["text"]  )

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(__snake_case				:       Optional[int]  ):
              # Concatenate all texts.
              lowercase    :				List[Any]       		=							{k: sum(examples[k]     , []  ) for k in examples.keys()}
              lowercase    :				Any       		=							len(concatenated_examples[list(examples.keys()  )[0]]  )
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              lowercase    :				Optional[Any]       		=							(total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              lowercase    :				int       		=							{
                  k: [t[i : i + args.max_length] for i in range(0     , __snake_case     , args.max_length  )]
                  for k, t in concatenated_examples.items()
              }
              return result

       lowercase    :				int       		=							dataset_tokenized.map(__snake_case     , batched=__snake_case     , batch_size=1000     , num_proc=4  )

       lowercase    :				Dict       		=							0
       lowercase    :				Union[str, Any]       		=							0
       for shard in range(0     , len(__snake_case  )     , args.shard_size  ):
              lowercase    :				Optional[int]       		=							grouped_dataset[shard : shard + args.shard_size]
              lowercase    :				str       		=							len(dataset_snapshot["input_ids"]  )
              lowercase    :				Tuple       		=							os.path.join(__snake_case     , f"""dataset-{shard_count}-{records_containing}.tfrecord"""  )
              lowercase    :				Optional[int]       		=							get_serialized_examples(__snake_case  )

              with tf.io.TFRecordWriter(__snake_case  ) as out_file:
                     for i in range(len(__snake_case  )  ):
                            lowercase    :				Union[str, Any]       		=							serialized_examples[i]
                            out_file.write(__snake_case  )
                     print("Wrote file {} containing {} records".format(__snake_case     , __snake_case  )  )

              shard_count += 1
              total_records += records_containing

       with open(f"""split-{args.split}-records-count.txt"""     , "w"  ) as f:
              print(f"""Total {args.split} records: {total_records}"""     , file=__snake_case  )


if __name__ == "__main__":
   _A			:       Tuple   		=       parse_args()
   main(args)




"""simple docstring"""




from __future__ import annotations

from functools import lru_cache
from math import ceil

_A			:       Optional[Any]   		=       1_00

_A			:       Optional[int]   		=       set(range(3, NUM_PRIMES, 2))
primes.add(2)
_A			:       int

for prime in range(3, ceil(NUM_PRIMES**0.5), 2):
   if prime not in primes:
      continue
   primes.difference_update(set(range(prime * prime, NUM_PRIMES, prime)))


@lru_cache(maxsize=100  )
def   __magic_name__	(      __snake_case				:       int  ) ->				set[int]:
       if number_to_partition < 0:
              return set()
       elif number_to_partition == 0:
              return {1}

       lowercase    :				set[int]       		=							set()
       lowercase    :				int
       lowercase    :				int

       for prime in primes:
              if prime > number_to_partition:
                     continue
              for sub in partition(number_to_partition - prime  ):
                     ret.add(sub * prime  )

       return ret


def   __magic_name__	(      __snake_case				:       int = 5000  ) ->				int | None:
       for number_to_partition in range(1     , __snake_case  ):
              if len(partition(__snake_case  )  ) > number_unique_partitions:
                     return number_to_partition
       return None


if __name__ == "__main__":
   print(F"{solution() = }")


