




'''simple docstring'''




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a_      :		Dict							      =       logging.getLogger(__name__)





def      a_							(      )							->						int:



       """simple docstring"""
       lowerCamelCase_							=argparse.ArgumentParser(
           description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''   )
       parser.add_argument(
           '''--dataset_name'''   ,       type=__snake_case   ,       default='''wikitext'''   ,       help='''Name of the training. Explore datasets at: hf.co/datasets.'''   ,       )
       parser.add_argument(
           '''--dataset_config'''   ,       type=__snake_case   ,       default='''wikitext-103-raw-v1'''   ,       help='''Configuration name of the dataset.'''   )
       parser.add_argument(
           '''--tokenizer_name_or_path'''   ,       type=__snake_case   ,       default='''sayakpaul/unigram-tokenizer-wikitext'''   ,       help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''   ,       )
       parser.add_argument(
           '''--shard_size'''   ,       type=__snake_case   ,       default=1000   ,       help='''Number of entries to go in a single shard.'''   ,       )
       parser.add_argument('''--split'''   ,       type=__snake_case   ,       default='''train'''   ,       choices=['''train''', '''test''', '''validation''']   )
       parser.add_argument(
           '''--limit'''   ,       default=__snake_case   ,       type=__snake_case   ,       help='''Limit the number of shards (used for debugging).'''   ,       )
       parser.add_argument(
           '''--max_length'''   ,       type=__snake_case   ,       default=512   ,       help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
           ''' sequence length that is a multiple of 8.'''   ,       )
       parser.add_argument(
           '''--output_dir'''   ,       default='''tf-tpu'''   ,       type=__snake_case   ,       help='''Output directory where the TFRecord shards will be saved. If the'''
           ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
           ''' shards will be directly saved to a Google Cloud Storage bucket.'''   ,       )

       lowerCamelCase_							=parser.parse_args()
       return args





def      a_							(      __snake_case							:			Optional[int]   )							->						Union[str, Any]:



       """simple docstring"""





       def fn(__snake_case							:			Tuple   ):
              return tokenizer(examples['''text''']   )

       return fn





def      a_							(      __snake_case							:			Any   )							->						int:



       """simple docstring"""
       lowerCamelCase_							=[]
       for i in range(len(tokenized_data['''input_ids''']   )   ):
              lowerCamelCase_							={
                  '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]   )   ),
                  '''attention_mask''': tf.train.Feature(
                      intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]   )   ),
              }
              lowerCamelCase_							=tf.train.Features(feature=__snake_case   )
              lowerCamelCase_							=tf.train.Example(features=__snake_case   )
              lowerCamelCase_							=example.SerializeToString()
              records.append(__snake_case   )
       return records





def      a_							(      __snake_case							:			Union[str, Any]   )							->						Dict:



       """simple docstring"""
       lowerCamelCase_							=datasets.load_dataset(args.dataset_name   ,       args.dataset_config   ,       split=args.split   )

       if args.limit is not None:
              lowerCamelCase_							=min(len(__snake_case   )   ,       args.limit   )
              lowerCamelCase_							=dataset.select(range(__snake_case   )   )
              print(F'''Limiting the dataset to {args.limit} entries.'''   )

       lowerCamelCase_							=AutoTokenizer.from_pretrained(args.tokenizer_name_or_path   )

       # Handle output directory creation.
       # For serializing into a Google Cloud Storage Bucket, one needs to first
       # create a bucket.
       if "gs" not in args.output_dir:
              if not os.path.exists(args.output_dir   ):
                     os.makedirs(args.output_dir   )
              lowerCamelCase_							=os.path.join(args.output_dir   ,       args.split   )
              if not os.path.exists(__snake_case   ):
                     os.makedirs(__snake_case   )
       else:
              lowerCamelCase_							=os.path.join(args.output_dir   ,       args.split   )

       # Tokenize the whole dataset at once.
       lowerCamelCase_							=tokenize_function(__snake_case   )
       lowerCamelCase_							=dataset.map(__snake_case   ,       batched=__snake_case   ,       num_proc=4   ,       remove_columns=['''text''']   )

       # We need to concatenate all our texts together, and then split the result
       # into chunks of a fixed size, which we will call block_size. To do this, we
       # will use the map method again, with the option batched=True. When we use batched=True,
       # the function we pass to map() will be passed multiple inputs at once, allowing us
       # to group them into more or fewer examples than we had in the input.
       # This allows us to create our new fixed-length samples. The advantage of this
       # method is that we don't lose a whole lot of content from the dataset compared to the
       # case where we simply tokenize with a pre-defined max_length.

       def group_texts(__snake_case							:			str   ):
              # Concatenate all texts.
              lowerCamelCase_							={k: sum(examples[k]   ,       []   ) for k in examples.keys()}
              lowerCamelCase_							=len(concatenated_examples[list(examples.keys()   )[0]]   )
              # We drop the small remainder, though you could add padding instead if the model supports it
              # In this, as in all things, we advise you to follow your heart 🫀
              lowerCamelCase_							=(total_length // args.max_length) * args.max_length
              # Split by chunks of max_len.
              lowerCamelCase_							={
                  k: [t[i : i + args.max_length] for i in range(0   ,       __snake_case   ,       args.max_length   )]
                  for k, t in concatenated_examples.items()
              }
              return result

       lowerCamelCase_							=dataset_tokenized.map(__snake_case   ,       batched=__snake_case   ,       batch_size=1000   ,       num_proc=4   )

       lowerCamelCase_							=0
       lowerCamelCase_							=0
       for shard in range(0   ,       len(__snake_case   )   ,       args.shard_size   ):
              lowerCamelCase_							=grouped_dataset[shard : shard + args.shard_size]
              lowerCamelCase_							=len(dataset_snapshot['''input_ids''']   )
              lowerCamelCase_							=os.path.join(__snake_case   ,       F'''dataset-{shard_count}-{records_containing}.tfrecord'''   )
              lowerCamelCase_							=get_serialized_examples(__snake_case   )

              with tf.io.TFRecordWriter(__snake_case   ) as out_file:
                     for i in range(len(__snake_case   )   ):
                            lowerCamelCase_							=serialized_examples[i]
                            out_file.write(__snake_case   )
                     print('''Wrote file {} containing {} records'''.format(__snake_case   ,       __snake_case   )   )

              shard_count += 1
              total_records += records_containing

       with open(F'''split-{args.split}-records-count.txt'''   ,       '''w'''   ) as f:
              print(F'''Total {args.split} records: {total_records}'''   ,       file=__snake_case   )


if __name__ == "__main__":
     a_      :		Optional[Any]							      =       parse_args()
     main(args)












"""simple docstring"""



def 						_SCREAMING_SNAKE_CASE (    __snake_case			:  int    ):





  '''simple docstring'''





  return sum(i for i in range(1		,					number // 2 + 1    ) if number % i == 0    ) == number


if __name__ == "__main__":
    print('Program to check whether a number is a Perfect number or not...')
    _UpperCamelCase		:  Tuple						=		int(input('Enter number: ').strip())
    print(F'''{number} is {'' if perfect(number) else 'not '}a Perfect Number.''')


