code: 
  
  
  import os
  import unittest
  
  from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  class 				__SCREAMING_SNAKE_CASE   (			A__  ,						unittest.TestCase   ):
        A				:							List[Any]    							=     BertweetTokenizer
        A				:							int    							=     False
  
  
        def        __lowerCamelCase			(							self		):
              super().setUp()
  
              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              lowercase			:   Optional[int]					   =		['''I''', '''m''', '''V@@''', '''R@@''', '''r''', '''e@@''']
              lowercase			:   Union[str, Any]					   =		dict(zip(SCREAMING_SNAKE_CASE__							,						range(len(SCREAMING_SNAKE_CASE__		)		)		)		)
              lowercase			:   int					   =		['''#version: 0.2''', '''a m</w>''']
              lowercase			:   Union[str, Any]					   =		{'''unk_token''': '''<unk>'''}
  
              lowercase			:   List[Any]					   =		os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file''']		)
              lowercase			:   Dict					   =		os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''merges_file''']		)
              with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8'''		) as fp:
                    for token in vocab_tokens:
                          fp.write(f"""{token} {vocab_tokens[token]}\n"""		)
              with open(self.merges_file							,						'''w'''							,						encoding='''utf-8'''		) as fp:
                    fp.write('''\n'''.join(SCREAMING_SNAKE_CASE__		)		)
  
  
        def        __lowerCamelCase			(							self							,						**SCREAMING_SNAKE_CASE__		):
              kwargs.update(self.special_tokens_map		)
              return BertweetTokenizer.from_pretrained(self.tmpdirname							,						**SCREAMING_SNAKE_CASE__		)
  
  
        def        __lowerCamelCase			(							self							,						SCREAMING_SNAKE_CASE__		):
              lowercase			:   List[Any]					   =		'''I am VinAI Research'''
              lowercase			:   Optional[int]					   =		'''I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
              return input_text, output_text
  
  
  
  
  
  
        def        __lowerCamelCase			(							self		):
              lowercase			:   Optional[Any]					   =		BertweetTokenizer(self.vocab_file							,						self.merges_file							,						**self.special_tokens_map		)
              lowercase			:   str					   =		'''I am VinAI Research'''
              lowercase			:   Tuple					   =		'''I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
              lowercase			:   Optional[int]					   =		tokenizer.tokenize(SCREAMING_SNAKE_CASE__		)
              self.assertListEqual(SCREAMING_SNAKE_CASE__							,						SCREAMING_SNAKE_CASE__		)
  
              lowercase			:   str					   =		tokens + [tokenizer.unk_token]
  
              lowercase			:   int					   =		[4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__		)							,						SCREAMING_SNAKE_CASE__		)
  
  
style_context: 
  
  
  import argparse
  import logging
  
  import numpy as np
  import torch
  
  from transformers import AutoModelForCausalLM, AutoTokenizer
  
  
  logging.basicConfig(
      format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''',
      datefmt='''%m/%d/%Y %H:%M:%S''',
      level=logging.INFO,
  )
  __a   				=      logging.getLogger(__name__)
  
  
  
  
  def 			__lowercase	(						_UpperCamelCase     )    ->Any:
  
  
  
  
  
  
  
        """simple docstring"""
  
        np.random.seed(args.seed     )
        torch.manual_seed(args.seed     )
        if args.n_gpu > 0:
              torch.cuda.manual_seed_all(args.seed     )
  
  
  
  
  def 			__lowercase	(						)    ->List[str]:
  
  
  
  
  
  
  
        """simple docstring"""
  
        lowercase			:   List[str]					   =		argparse.ArgumentParser()
        parser.add_argument(
            '''--model_name_or_path''',    default=_UpperCamelCase,    type=_UpperCamelCase,    required=_UpperCamelCase,    )
        parser.add_argument('''--prompt''',    type=_UpperCamelCase,    default=''''''     )
        parser.add_argument('''--length''',    type=_UpperCamelCase,    default=20     )
        parser.add_argument('''--stop_token''',    type=_UpperCamelCase,    default=_UpperCamelCase,    help='''Token at which text generation is stopped'''     )
        parser.add_argument(
            '''--temperature''',    type=_UpperCamelCase,    default=1.0,    help='''temperature of 1.0 has no effect, lower tend toward greedy sampling''',    )
        parser.add_argument(
            '''--repetition_penalty''',    type=_UpperCamelCase,    default=1.0,    help='''primarily useful for CTRL model; in that case, use 1.2'''     )
        parser.add_argument('''--k''',    type=_UpperCamelCase,    default=0     )
        parser.add_argument('''--penalty_alpha''',    type=_UpperCamelCase,    default=0.0     )
        parser.add_argument('''--p''',    type=_UpperCamelCase,    default=0.9     )
  
        parser.add_argument('''--prefix''',    type=_UpperCamelCase,    default='''''',    help='''Text added prior to input.'''     )
        parser.add_argument('''--padding_text''',    type=_UpperCamelCase,    default='''''',    help='''Deprecated, the use of `--prefix` is preferred.'''     )
        parser.add_argument('''--xlm_language''',    type=_UpperCamelCase,    default='''''',    help='''Optional language when used with the XLM model.'''     )
  
        parser.add_argument('''--seed''',    type=_UpperCamelCase,    default=42,    help='''random seed for initialization'''     )
        parser.add_argument('''--no_cuda''',    action='''store_true''',    help='''Avoid using CUDA when available'''     )
        parser.add_argument(
            '''--fp16''',    action='''store_true''',    help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit''',    )
        lowercase			:   Any					   =		parser.parse_args()
  
        lowercase			:   int					   =		torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu'''     )
        lowercase			:   List[str]					   =		0 if args.no_cuda else torch.cuda.device_count()
  
        logger.warning(f"""device: {args.device}, n_gpu: {args.n_gpu}, 16-bits training: {args.fpaa}"""     )
  
        set_seed(_UpperCamelCase     )
  
        # Initialize the model and tokenizer
        lowercase			:   Optional[Any]					   =		AutoTokenizer.from_pretrained(args.model_name_or_path     )
        lowercase			:   List[Any]					   =		AutoModelForCausalLM.from_pretrained(args.model_name_or_path     )
  
        # tokenizer = GPT2Tokenizer.from_pretrained(args.model_name_or_path)
        # model = OPTForCausalLM.from_pretrained(args.model_name_or_path)
        model.to(args.device     )
  
        if args.fpaa:
              model.half()
  
        logger.info(_UpperCamelCase     )
        lowercase			:   Union[str, Any]					   =		args.prompt if args.prompt else input('''Model prompt >>> '''     )
  
        lowercase			:   List[Any]					   =		tokenizer(_UpperCamelCase,    return_tensors='''pt''',    add_special_tokens=_UpperCamelCase     )
        lowercase			:   Union[str, Any]					   =		{key: value.to(args.device     ) for key, value in inputs.items()}
  
        lowercase			:   Optional[int]					   =		model.generate(
            **_UpperCamelCase,    max_length=args.length + len(inputs['''input_ids'''][0]     ),    penalty_alpha=args.penalty_alpha,    top_k=args.k,    )
  
        lowercase			:   Union[str, Any]					   =		[]
        for generated_sequence_idx, generated_sequence in enumerate(_UpperCamelCase     ):
              print(f"""=== GENERATED SEQUENCE {generated_sequence_idx + 1} ==="""     )
              lowercase			:   Optional[Any]					   =		generated_sequence.tolist()
  
              # Decode text
              lowercase			:   int					   =		tokenizer.decode(_UpperCamelCase,    clean_up_tokenization_spaces=_UpperCamelCase,    add_special_tokens=_UpperCamelCase     )
  
              # Remove all text after the stop token
              lowercase			:   Optional[int]					   =		text[: text.find(args.stop_token     ) if args.stop_token else None]
  
              # Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing
              lowercase			:   Optional[Any]					   =		(
                  prompt_text + text[len(tokenizer.decode(inputs['''input_ids'''][0],    clean_up_tokenization_spaces=_UpperCamelCase     )     ) :]
              )
  
              generated_sequences.append(_UpperCamelCase     )
              print(_UpperCamelCase     )
  
        return generated_sequences
  
  
  if __name__ == "__main__":
    main()
  
  