




'''simple docstring'''




from __future__ import annotations

import random

# Maximum size of the population.  Bigger could be faster but is more memory expensive.
a_      :		str							      =       2_00
# Number of elements selected in every generation of evolution. The selection takes
# place from best to worst of that generation and must be smaller than N_POPULATION.
a_      :		Tuple							      =       50
# Probability that an element of a generation can mutate, changing one of its genes.
# This will guarantee that all genes will be used during evolution.
a_      :		List[Any]							      =       0.4
# Just a seed to improve randomness required by the algorithm.
random.seed(random.randint(0, 10_00))





def      a_							(      __snake_case							:			str   ,       __snake_case							:			str   )							->						tuple[str, float]:



       """simple docstring"""
       lowerCamelCase_							=len([g for position, g in enumerate(__snake_case   ) if g == main_target[position]]   )
       return (item, float(__snake_case   ))





def      a_							(      __snake_case							:			str   ,       __snake_case							:			str   )							->						tuple[str, str]:



       """simple docstring"""
       lowerCamelCase_							=random.randint(0   ,       len(__snake_case   ) - 1   )
       lowerCamelCase_							=parent_a[:random_slice] + parent_a[random_slice:]
       lowerCamelCase_							=parent_a[:random_slice] + parent_a[random_slice:]
       return (child_a, child_a)





def      a_							(      __snake_case							:			str   ,       __snake_case							:			list[str]   )							->						str:



       """simple docstring"""
       lowerCamelCase_							=list(__snake_case   )
       if random.uniform(0   ,       1   ) < MUTATION_PROBABILITY:
              lowerCamelCase_							=random.choice(__snake_case   )
       return "".join(__snake_case   )





def      a_							(      __snake_case							:			tuple[str, float]   ,       __snake_case							:			list[tuple[str, float]]   ,       __snake_case							:			list[str]   ,       )							->						list[str]:



       """simple docstring"""
       lowerCamelCase_							=[]
       # Generate more children proportionally to the fitness score.
       lowerCamelCase_							=int(parent_a[1] * 100   ) + 1
       lowerCamelCase_							=10 if child_n >= 10 else child_n
       for _ in range(__snake_case   ):
              lowerCamelCase_							=population_score[random.randint(0   ,       __snake_case   )][0]

              lowerCamelCase_,      lowerCamelCase_							=crossover(parent_a[0]   ,       __snake_case   )
              # Append new string to the population list.
              pop.append(mutate(__snake_case   ,       __snake_case   )   )
              pop.append(mutate(__snake_case   ,       __snake_case   )   )
       return pop





def      a_							(      __snake_case							:			str   ,       __snake_case							:			list[str]   ,       __snake_case							:			bool = True   )							->						tuple[int, int, str]:



       """simple docstring"""

       # Verify if N_POPULATION is bigger than N_SELECTED
       if N_POPULATION < N_SELECTED:
              lowerCamelCase_							=F'''{N_POPULATION} must be bigger than {N_SELECTED}'''
              raise ValueError(__snake_case   )
       # Verify that the target contains no genes besides the ones inside genes variable.
       lowerCamelCase_							=sorted({c for c in target if c not in genes}   )
       if not_in_genes_list:
              lowerCamelCase_							=F'''{not_in_genes_list} is not in genes list, evolution cannot converge'''
              raise ValueError(__snake_case   )

       # Generate random starting population.
       lowerCamelCase_							=[]
       for _ in range(__snake_case   ):
              population.append(''''''.join([random.choice(__snake_case   ) for i in range(len(__snake_case   )   )]   )   )

       # Just some logs to know what the algorithms is doing.
       lowerCamelCase_,      lowerCamelCase_							=0, 0

       # This loop will end when we find a perfect match for our target.
       while True:
              generation += 1
              total_population += len(__snake_case   )

              # Random population created. Now it's time to evaluate.

              # Adding a bit of concurrency can make everything faster,
              #
              # import concurrent.futures
              # population_score: list[tuple[str, float]] = []
              # with concurrent.futures.ThreadPoolExecutor(
              #                                   max_workers=NUM_WORKERS) as executor:
              #     futures = {executor.submit(evaluate, item) for item in population}
              #     concurrent.futures.wait(futures)
              #     population_score = [item.result() for item in futures]
              #
              # but with a simple algorithm like this, it will probably be slower.
              # We just need to call evaluate for every item inside the population.
              lowerCamelCase_							=[evaluate(__snake_case   ,       __snake_case   ) for item in population]

              # Check if there is a matching evolution.
              lowerCamelCase_							=sorted(__snake_case   ,       key=lambda __snake_case   : x[1]   ,       reverse=__snake_case   )
              if population_score[0][0] == target:
                     return (generation, total_population, population_score[0][0])

              # Print the best result every 10 generation.
              # Just to know that the algorithm is working.
              if debug and generation % 10 == 0:
                     print(
                         F'''\nGeneration: {generation}'''
                         F'''\nTotal Population:{total_population}'''
                         F'''\nBest score: {population_score[0][1]}'''
                         F'''\nBest string: {population_score[0][0]}'''   )

              # Flush the old population, keeping some of the best evolutions.
              # Keeping this avoid regression of evolution.
              lowerCamelCase_							=population[: int(N_POPULATION / 3   )]
              population.clear()
              population.extend(__snake_case   )
              # Normalize population score to be between 0 and 1.
              lowerCamelCase_							=[
                  (item, score / len(__snake_case   )) for item, score in population_score
              ]

              # This is selection
              for i in range(__snake_case   ):
                     population.extend(select(population_score[int(__snake_case   )]   ,       __snake_case   ,       __snake_case   )   )
                     # Check if the population has already reached the maximum value and if so,
                     # break the cycle.  If this check is disabled, the algorithm will take
                     # forever to compute large strings, but will also calculate small strings in
                     # a far fewer generations.
                     if len(__snake_case   ) > N_POPULATION:
                            break


if __name__ == "__main__":
     a_      :		str							      =       (
         """This is a genetic algorithm to evaluate, combine, evolve, and mutate a string!"""
     )
     a_      :		Dict							      =       list(
         """ ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklm"""
         """nopqrstuvwxyz.,;!?+-*#@^'èéòà€ù=)(&%$£/\\"""
     )
     a_      ,       a_      ,       a_      :		Optional[Any]							      =       basic(target_str, genes_list)
     print(
         F"""\nGeneration: {generation}\nTotal Population: {population}\nTarget: {target}"""
     )









"""simple docstring"""







import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowercase__		:				List[Any]          =	logging.getLogger(__name__)

def 				UpperCamelCase_    (    )						->   Dict:

  """simple docstring"""




  lowerCAmelCase_   :				Tuple								= argparse.ArgumentParser(
      description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'						)
  parser.add_argument(
      '--dataset_name'    ,   type=lowerCAmelCase__    ,   default='wikitext'    ,   help='Name of the training. Explore datasets at: hf.co/datasets.'    ,   )
  parser.add_argument(
      '--dataset_config'    ,   type=lowerCAmelCase__    ,   default='wikitext-103-raw-v1'    ,   help='Configuration name of the dataset.'						)
  parser.add_argument(
      '--tokenizer_name_or_path'    ,   type=lowerCAmelCase__    ,   default='sayakpaul/unigram-tokenizer-wikitext'    ,   help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'    ,   )
  parser.add_argument(
      '--shard_size'    ,   type=lowerCAmelCase__    ,   default=1000    ,   help='Number of entries to go in a single shard.'    ,   )
  parser.add_argument('--split'    ,   type=lowerCAmelCase__    ,   default='train'    ,   choices=['train', 'test', 'validation']						)
  parser.add_argument(
      '--limit'    ,   default=lowerCAmelCase__    ,   type=lowerCAmelCase__    ,   help='Limit the number of shards (used for debugging).'    ,   )
  parser.add_argument(
      '--max_length'    ,   type=lowerCAmelCase__    ,   default=512    ,   help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
      ' sequence length that is a multiple of 8.'    ,   )
  parser.add_argument(
      '--output_dir'    ,   default='tf-tpu'    ,   type=lowerCAmelCase__    ,   help='Output directory where the TFRecord shards will be saved. If the'
      ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
      ' shards will be directly saved to a Google Cloud Storage bucket.'    ,   )

  lowerCAmelCase_   :				List[Any]								= parser.parse_args()
  return args

def 				UpperCamelCase_    (    lowerCAmelCase__      :				int						)						->   Union[str, Any]:

  """simple docstring"""




  def fn(lowerCAmelCase__      :				Optional[Any]						):
    return tokenizer(examples['text']						)

  return fn

def 				UpperCamelCase_    (    lowerCAmelCase__      :				Tuple						)						->   Dict:

  """simple docstring"""




  lowerCAmelCase_   :				int								= []
  for i in range(len(tokenized_data['input_ids']						)						):
    lowerCAmelCase_   :				Tuple								= {
        'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]						)						),
        'attention_mask': tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]						)						),
    }
    lowerCAmelCase_   :				Union[str, Any]								= tf.train.Features(feature=lowerCAmelCase__						)
    lowerCAmelCase_   :				Dict								= tf.train.Example(features=lowerCAmelCase__						)
    lowerCAmelCase_   :				Optional[int]								= example.SerializeToString()
    records.append(lowerCAmelCase__						)
  return records

def 				UpperCamelCase_    (    lowerCAmelCase__      :				Optional[Any]						)						->   Optional[int]:

  """simple docstring"""




  lowerCAmelCase_   :				str								= datasets.load_dataset(args.dataset_name    ,   args.dataset_config    ,   split=args.split						)

  if args.limit is not None:
    lowerCAmelCase_   :				Tuple								= min(len(lowerCAmelCase__						)    ,   args.limit						)
    lowerCAmelCase_   :				Any								= dataset.select(range(lowerCAmelCase__						)						)
    print(f"Limiting the dataset to {args.limit} entries."						)

  lowerCAmelCase_   :				List[str]								= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir						):
      os.makedirs(args.output_dir						)
    lowerCAmelCase_   :				int								= os.path.join(args.output_dir    ,   args.split						)
    if not os.path.exists(lowerCAmelCase__						):
      os.makedirs(lowerCAmelCase__						)
  else:
    lowerCAmelCase_   :				Dict								= os.path.join(args.output_dir    ,   args.split						)

  # Tokenize the whole dataset at once.
  lowerCAmelCase_   :				Dict								= tokenize_function(lowerCAmelCase__						)
  lowerCAmelCase_   :				List[Any]								= dataset.map(lowerCAmelCase__    ,   batched=lowerCAmelCase__    ,   num_proc=4    ,   remove_columns=['text']						)

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(lowerCAmelCase__      :				List[Any]						):
    # Concatenate all texts.
    lowerCAmelCase_   :				int								= {k: sum(examples[k]    ,   []						) for k in examples.keys()}
    lowerCAmelCase_   :				Any								= len(concatenated_examples[list(examples.keys()						)[0]]						)
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    lowerCAmelCase_   :				Union[str, Any]								= (total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    lowerCAmelCase_   :				Optional[Any]								= {
        k: [t[i : i + args.max_length] for i in range(0    ,   lowerCAmelCase__    ,   args.max_length						)]
        for k, t in concatenated_examples.items()
    }
    return result

  lowerCAmelCase_   :				Optional[int]								= dataset_tokenized.map(lowerCAmelCase__    ,   batched=lowerCAmelCase__    ,   batch_size=1000    ,   num_proc=4						)

  lowerCAmelCase_   :				Optional[int]								= 0
  lowerCAmelCase_   :				Optional[Any]								= 0
  for shard in range(0    ,   len(lowerCAmelCase__						)    ,   args.shard_size						):
    lowerCAmelCase_   :				Dict								= grouped_dataset[shard : shard + args.shard_size]
    lowerCAmelCase_   :				Tuple								= len(dataset_snapshot['input_ids']						)
    lowerCAmelCase_   :				Optional[Any]								= os.path.join(lowerCAmelCase__    ,   f"dataset-{shard_count}-{records_containing}.tfrecord"						)
    lowerCAmelCase_   :				Tuple								= get_serialized_examples(lowerCAmelCase__						)

    with tf.io.TFRecordWriter(lowerCAmelCase__						) as out_file:
      for i in range(len(lowerCAmelCase__						)						):
        lowerCAmelCase_   :				Dict								= serialized_examples[i]
        out_file.write(lowerCAmelCase__						)
      print('Wrote file {} containing {} records'.format(lowerCAmelCase__    ,   lowerCAmelCase__						)						)

    shard_count += 1
    total_records += records_containing

  with open(f"split-{args.split}-records-count.txt"    ,   'w'						) as f:
    print(f"Total {args.split} records: {total_records}"    ,   file=lowerCAmelCase__						)


if __name__ == "__main__":
     lowercase__		:				int          =	parse_args()
     main(args)





