



"""simple docstring"""







import argparse
import logging

import numpy as np
import torch

from transformers import AutoModelForCausalLM, AutoTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""",
    datefmt="""%m/%d/%Y %H:%M:%S""",
    level=logging.INFO,
)
UpperCAmelCase:  Any			     =				logging.getLogger(__name__)


def  __SCREAMING_SNAKE_CASE (  __UpperCAmelCase      ):
		np.random.seed(args.seed      )
		torch.manual_seed(args.seed      )
		if args.n_gpu > 0:
				torch.cuda.manual_seed_all(args.seed      )


def  __SCREAMING_SNAKE_CASE (  ):
		_lowercase				: Any     =   argparse.ArgumentParser()
		parser.add_argument(
		    """--model_name_or_path"""   ,				default=__UpperCAmelCase   ,				type=__UpperCAmelCase   ,				required=__UpperCAmelCase   ,				)
		parser.add_argument("""--prompt"""   ,				type=__UpperCAmelCase   ,				default=""""""      )
		parser.add_argument("""--length"""   ,				type=__UpperCAmelCase   ,				default=20      )
		parser.add_argument("""--stop_token"""   ,				type=__UpperCAmelCase   ,				default=__UpperCAmelCase   ,				help="""Token at which text generation is stopped"""      )
		parser.add_argument(
		    """--temperature"""   ,				type=__UpperCAmelCase   ,				default=1.0   ,				help="""temperature of 1.0 has no effect, lower tend toward greedy sampling"""   ,				)
		parser.add_argument(
		    """--repetition_penalty"""   ,				type=__UpperCAmelCase   ,				default=1.0   ,				help="""primarily useful for CTRL model; in that case, use 1.2"""      )
		parser.add_argument("""--k"""   ,				type=__UpperCAmelCase   ,				default=0      )
		parser.add_argument("""--penalty_alpha"""   ,				type=__UpperCAmelCase   ,				default=0.0      )
		parser.add_argument("""--p"""   ,				type=__UpperCAmelCase   ,				default=0.9      )

		parser.add_argument("""--prefix"""   ,				type=__UpperCAmelCase   ,				default=""""""   ,				help="""Text added prior to input."""      )
		parser.add_argument("""--padding_text"""   ,				type=__UpperCAmelCase   ,				default=""""""   ,				help="""Deprecated, the use of `--prefix` is preferred."""      )
		parser.add_argument("""--xlm_language"""   ,				type=__UpperCAmelCase   ,				default=""""""   ,				help="""Optional language when used with the XLM model."""      )

		parser.add_argument("""--seed"""   ,				type=__UpperCAmelCase   ,				default=42   ,				help="""random seed for initialization"""      )
		parser.add_argument("""--no_cuda"""   ,				action="""store_true"""   ,				help="""Avoid using CUDA when available"""      )
		parser.add_argument(
		    """--fp16"""   ,				action="""store_true"""   ,				help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""   ,				)
		_lowercase				: str     =   parser.parse_args()

		_lowercase				: str     =   torch.device("""cuda""" if torch.cuda.is_available() and not args.no_cuda else """cpu"""      )
		_lowercase				: List[str]     =   0 if args.no_cuda else torch.cuda.device_count()

		logger.warning(F"""device: {args.device}, n_gpu: {args.n_gpu}, 16-bits training: {args.fpaa}"""      )

		set_seed(__UpperCAmelCase      )

		# Initialize the model and tokenizer
		_lowercase				: List[str]     =   AutoTokenizer.from_pretrained(args.model_name_or_path      )
		_lowercase				: Union[str, Any]     =   AutoModelForCausalLM.from_pretrained(args.model_name_or_path      )

		# tokenizer = GPT2Tokenizer.from_pretrained(args.model_name_or_path)
		# model = OPTForCausalLM.from_pretrained(args.model_name_or_path)
		model.to(args.device      )

		if args.fpaa:
				model.half()

		logger.info(__UpperCAmelCase      )
		_lowercase				: Any     =   args.prompt if args.prompt else input("""Model prompt >>> """      )

		_lowercase				: Optional[Any]     =   tokenizer(__UpperCAmelCase   ,				return_tensors="""pt"""   ,				add_special_tokens=__UpperCAmelCase      )
		_lowercase				: Dict     =   {key: value.to(args.device      ) for key, value in inputs.items()}

		_lowercase				: Optional[Any]     =   model.generate(
		    **__UpperCAmelCase   ,				max_length=args.length + len(inputs["""input_ids"""][0]      )   ,				penalty_alpha=args.penalty_alpha   ,				top_k=args.k   ,				)

		_lowercase				: Tuple     =   []
		for generated_sequence_idx, generated_sequence in enumerate(__UpperCAmelCase      ):
				print(F"""=== GENERATED SEQUENCE {generated_sequence_idx + 1} ==="""      )
				_lowercase				: List[Any]     =   generated_sequence.tolist()

				# Decode text
				_lowercase				: Optional[int]     =   tokenizer.decode(__UpperCAmelCase   ,				clean_up_tokenization_spaces=__UpperCAmelCase   ,				add_special_tokens=__UpperCAmelCase      )

				# Remove all text after the stop token
				_lowercase				: Dict     =   text[: text.find(args.stop_token      ) if args.stop_token else None]

				# Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing
				_lowercase				: Tuple     =   (
				    prompt_text + text[len(tokenizer.decode(inputs["""input_ids"""][0]   ,				clean_up_tokenization_spaces=__UpperCAmelCase      )      ) :]
				)

				generated_sequences.append(__UpperCAmelCase      )
				print(__UpperCAmelCase      )

		return generated_sequences


if __name__ == "__main__":
		main()








"""simple docstring"""







import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCAmelCase:  List[str]			     =				logging.getLogger(__name__)


def  __SCREAMING_SNAKE_CASE (  ):
		_lowercase				: List[Any]     =   argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""      )
		parser.add_argument(
		    """--dataset_name"""   ,				type=__UpperCAmelCase   ,				default="""wikitext"""   ,				help="""Name of the training. Explore datasets at: hf.co/datasets."""   ,				)
		parser.add_argument(
		    """--dataset_config"""   ,				type=__UpperCAmelCase   ,				default="""wikitext-103-raw-v1"""   ,				help="""Configuration name of the dataset."""      )
		parser.add_argument(
		    """--batch_size"""   ,				type=__UpperCAmelCase   ,				default=1000   ,				help="""Batch size during training."""   ,				)
		parser.add_argument(
		    """--vocab_size"""   ,				type=__UpperCAmelCase   ,				default=10048   ,				help="""Size of the desired vocabulary."""   ,				)
		parser.add_argument(
		    """--limit"""   ,				default=__UpperCAmelCase   ,				type=__UpperCAmelCase   ,				help="""Limit the number of shards (used for debugging)."""   ,				)
		parser.add_argument(
		    """--export_to_hub"""   ,				action="""store_true"""   ,				)

		_lowercase				: Any     =   parser.parse_args()
		return args


def  __SCREAMING_SNAKE_CASE (  __UpperCAmelCase      ):
		_lowercase				: List[Any]     =   datasets.load_dataset(args.dataset_name   ,				args.dataset_config   ,				split="""train"""      )

		if args.limit is not None:
				_lowercase				: Tuple     =   min(len(__UpperCAmelCase      )   ,				args.limit      )
				_lowercase				: Optional[Any]     =   dataset.select(range(__UpperCAmelCase      )      )
				logger.info(F"""Limiting the dataset to {args.limit} entries."""      )

		def batch_iterator():
				for i in range(0   ,				len(__UpperCAmelCase      )   ,				args.batch_size      ):
						yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
		_lowercase				: Optional[int]     =   Tokenizer(Unigram()      )
		_lowercase				: List[str]     =   normalizers.Sequence([normalizers.Replace("""``"""   ,				"""\""""      ), normalizers.Replace("""''"""   ,				"""\""""      )]      )
		_lowercase				: str     =   pre_tokenizers.Metaspace()

		# Prepare the trainer.
		_lowercase				: int     =   UnigramTrainer(
		    unk_token="""<unk>"""   ,				special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]   ,				vocab_size=args.vocab_size   ,				)

		logger.info("""Training the tokenizer."""      )
		tokenizer.train_from_iterator(batch_iterator()   ,				trainer=__UpperCAmelCase      )
		logger.info("""Tokenizer training complete!"""      )

		_lowercase				: int     =   tokenizer.token_to_id("""[CLS]"""      )
		_lowercase				: Optional[Any]     =   tokenizer.token_to_id("""[SEP]"""      )
		_lowercase				: Optional[int]     =   processors.TemplateProcessing(
		    single="""[CLS]:0 $A:0 [SEP]:0"""   ,				pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""   ,				special_tokens=[
		        ("""[CLS]""", cls_token_id),
		        ("""[SEP]""", sep_token_id),
		    ]   ,				)
		_lowercase				: Union[str, Any]     =   decoders.Metaspace()

		if args.export_to_hub:
				logger.info("""Exporting the trained tokenzier to Hub."""      )
				_lowercase				: List[Any]     =   AlbertTokenizerFast(tokenizer_object=__UpperCAmelCase      )
				new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""      )


if __name__ == "__main__":
		UpperCAmelCase:  List[Any]			     =				parse_args()
		main(args)



