




'''simple docstring'''

import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCamelCase_     			=       logging.getLogger(__name__)


def 	_UpperCAmelCase				(  )    ->						int:
  _lowerCAmelCase       :  Any   				=					argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""		)
  parser.add_argument(
      """--dataset_name"""  ,      type=_SCREAMING_SNAKE_CASE  ,      default="""wikitext"""  ,      help="""Name of the training. Explore datasets at: hf.co/datasets."""  ,      )
  parser.add_argument(
      """--dataset_config"""  ,      type=_SCREAMING_SNAKE_CASE  ,      default="""wikitext-103-raw-v1"""  ,      help="""Configuration name of the dataset."""		)
  parser.add_argument(
      """--batch_size"""  ,      type=_SCREAMING_SNAKE_CASE  ,      default=10_00  ,      help="""Batch size during training."""  ,      )
  parser.add_argument(
      """--vocab_size"""  ,      type=_SCREAMING_SNAKE_CASE  ,      default=1_00_48  ,      help="""Size of the desired vocabulary."""  ,      )
  parser.add_argument(
      """--limit"""  ,      default=_SCREAMING_SNAKE_CASE  ,      type=_SCREAMING_SNAKE_CASE  ,      help="""Limit the number of shards (used for debugging)."""  ,      )
  parser.add_argument(
      """--export_to_hub"""  ,      action="""store_true"""  ,      )

  _lowerCAmelCase       :  Any   				=					parser.parse_args()
  return args


def 	_UpperCAmelCase				(  _lowerCamelCase						:       int		)    ->						Dict:
  _lowerCAmelCase       :  Any   				=					datasets.load_dataset(args.dataset_name  ,      args.dataset_config  ,      split="""train"""		)

  if args.limit is not None:
    _lowerCAmelCase       :  Union[str, Any]   				=					min(len(_SCREAMING_SNAKE_CASE		)  ,      args.limit		)
    _lowerCAmelCase       :  int   				=					dataset.select(range(_SCREAMING_SNAKE_CASE		)		)
    logger.info(f'Limiting the dataset to {args.limit} entries.'		)

  def batch_iterator():
    for i in range(0  ,      len(_SCREAMING_SNAKE_CASE		)  ,      args.batch_size		):
      yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
  _lowerCAmelCase       :  List[Any]   				=					Tokenizer(Unigram()		)
  _lowerCAmelCase       :  Union[str, Any]   				=					normalizers.Sequence([normalizers.Replace("""``"""  ,      """\""""		), normalizers.Replace("""\'\'"""  ,      """\""""		)]		)
  _lowerCAmelCase       :  Optional[int]   				=					pre_tokenizers.Metaspace()

  # Prepare the trainer.
  _lowerCAmelCase       :  Dict   				=					UnigramTrainer(
      unk_token="""<unk>"""  ,      special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]  ,      vocab_size=args.vocab_size  ,      )

  logger.info("""Training the tokenizer."""		)
  tokenizer.train_from_iterator(batch_iterator()  ,      trainer=_SCREAMING_SNAKE_CASE		)
  logger.info("""Tokenizer training complete!"""		)

  _lowerCAmelCase       :  Optional[int]   				=					tokenizer.token_to_id("""[CLS]"""		)
  _lowerCAmelCase       :  int   				=					tokenizer.token_to_id("""[SEP]"""		)
  _lowerCAmelCase       :  List[Any]   				=					processors.TemplateProcessing(
      single="""[CLS]:0 $A:0 [SEP]:0"""  ,      pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""  ,      special_tokens=[
          ("""[CLS]""", cls_token_id),
          ("""[SEP]""", sep_token_id),
      ]  ,      )
  _lowerCAmelCase       :  int   				=					decoders.Metaspace()

  if args.export_to_hub:
    logger.info("""Exporting the trained tokenzier to Hub."""		)
    _lowerCAmelCase       :  Tuple   				=					AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE		)
    new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""		)


if __name__ == "__main__":
     UpperCamelCase_     			=       parse_args()
     main(args)




'''simple docstring'''

import os


def 	_UpperCAmelCase				(  )    ->						int:
  _lowerCAmelCase       :  Optional[int]   				=					os.path.dirname(os.path.realpath(_lowerCamelCase		)		)
  _lowerCAmelCase       :  Optional[Any]   				=					os.path.join(_lowerCamelCase  ,      """triangle.txt"""		)

  with open(_lowerCamelCase		) as in_file:
    _lowerCAmelCase       :  List[Any]   				=					[[int(_lowerCamelCase		) for i in line.split()] for line in in_file]

  while len(_lowerCamelCase		) != 1:
    _lowerCAmelCase       :  Any   				=					triangle.pop()
    _lowerCAmelCase       :  List[str]   				=					triangle[-1]
    for j in range(len(_lowerCamelCase		) - 1		):
      curr_row[j] += max(last_row[j]  ,      last_row[j + 1]		)
  return triangle[0][0]


if __name__ == "__main__":
     print(solution())


