code: 
  
  
  
  
  
  
  import argparse
  import time
  
  import torch
  import transformers
  from measures_util import end_measure, log_measures, start_measure
  from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer
  
  from accelerate.utils import compute_module_sizes
  
  
  lowerCamelCase			:							List[str]       =    {
      'gpt-j-6b': {'is_causal': True, 'model': 'sgugger/sharded-gpt-j-6B', 'tokenizer': 'EleutherAI/gpt-j-6B'},
      'gpt-neox': {'is_causal': True, 'model': 'EleutherAI/gpt-neox-20b'},
      'opt': {'is_causal': True, 'model': 'facebook/opt-30b'},
      'T0pp': {'is_causal': False, 'model': 'bigscience/T0pp', 'model_revision': 'sharded'},
  }
  
  lowerCamelCase			:							Any       =    [
      'Hello, my name is',
      'Are unicorns real? Unicorns are',
      'For the first time in several years,',
      'My name is Julien and I am',
      'The goal of life is',
      'Whenever I\'m sad, I like to',
  ]
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      )				->			List[Any]:
       UpperCamelCase__    :     Tuple        =						argparse.ArgumentParser(description="Run and time generations on a big model using Accelerate."    )
       parser.add_argument("model_name"      ,					type=lowercase__      ,					default=lowercase__      ,					help="The name of the model to try."    )
       parser.add_argument(
           "--tokenizer_name"      ,					type=lowercase__      ,					default=lowercase__      ,					help="The name of the tokenizer (if different from the model."    )
       parser.add_argument("--is_causal"      ,					type=lowercase__      ,					default=lowercase__      ,					help="Whether or not the model is causal."    )
       parser.add_argument(
           "--model_revision"      ,					type=lowercase__      ,					default=lowercase__      ,					help="The revision to use for the model checkpoint."    )
       parser.add_argument("--torch_dtype"      ,					type=lowercase__      ,					default=lowercase__      ,					help="The dtype for the model."    )
       parser.add_argument("--disk_offload"      ,					action="store_true"    )
  
       UpperCamelCase__    :     List[str]        =						parser.parse_args()
  
       # Sanitize args
       if args.model_name in DEFAULT_MODELS:
            UpperCamelCase__    :     Optional[int]        =						DEFAULT_MODELS[args.model_name]
            UpperCamelCase__    :     Tuple        =						defaults["model"]
            if args.tokenizer_name is None:
                 UpperCamelCase__    :     Any        =						defaults.get("tokenizer"      ,					args.model_name    )
            if args.is_causal is None:
                 UpperCamelCase__    :     List[str]        =						defaults["is_causal"]
            if args.model_revision is None:
                 UpperCamelCase__    :     str        =						defaults.get("model_revision"      ,					"main"    )
  
       if args.is_causal is None:
            raise ValueError("Could not infer the default for `--is_causal`, pass either True or False for it."    )
       if args.tokenizer_name is None:
            UpperCamelCase__    :     Union[str, Any]        =						args.model_name
       if args.model_revision is None:
            UpperCamelCase__    :     Tuple        =						"main"
  
       return args
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      )				->			Optional[int]:
       transformers.utils.logging.set_verbosity_error()
       UpperCamelCase__    :     Any        =						parse_args()
  
       if args.torch_dtype is None:
            UpperCamelCase__    :     str        =						AutoConfig.from_pretrained(args.model_name    )
            UpperCamelCase__    :     Union[str, Any]        =						getattr(lowercase__      ,					"torch_dtype"      ,					torch.floataa    )
       else:
            UpperCamelCase__    :     Tuple        =						getattr(lowercase__      ,					args.torch_dtype    )
       UpperCamelCase__    :     str        =						AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
       UpperCamelCase__    :     Optional[int]        =						{
           "torch_dtype": torch_dtype,
           "revision": args.model_revision,
       }
       if args.disk_offload:
            UpperCamelCase__    :     List[str]        =						"tmp_offload"
            UpperCamelCase__    :     List[str]        =						True
  
       UpperCamelCase__    :     Optional[int]        =						start_measure()
       UpperCamelCase__    :     List[str]        =						model_cls.from_pretrained(args.model_name      ,					device_map="auto"      ,					**lowercase__    )
       UpperCamelCase__    :     Optional[Any]        =						end_measure(lowercase__    )
       log_measures(lowercase__      ,					"Model loading"    )
  
       UpperCamelCase__    :     Any        =						compute_module_sizes(lowercase__    )
       UpperCamelCase__    :     Tuple        =						{v: 0 for v in model.hf_device_map.values()}
       for module, device in model.hf_device_map.items():
            device_size[device] += module_sizes[module]
       UpperCamelCase__    :     Any        =						"\n".join([f'- {device}: {size // 2**20}MiB' for device, size in device_size.items()]    )
       print(f'\nTheoretical use:\n{message}'    )
  
       UpperCamelCase__    :     Any        =						AutoTokenizer.from_pretrained(args.tokenizer_name    )
  
       UpperCamelCase__    :     List[str]        =						start_measure()
       UpperCamelCase__    :     Optional[Any]        =						[]
       UpperCamelCase__    :     int        =						[]
       UpperCamelCase__    :     Optional[Any]        =						[]
       for prompt in PROMPTS:
            UpperCamelCase__    :     Dict        =						tokenizer(lowercase__      ,					return_tensors="pt"    ).to(0    )
            UpperCamelCase__    :     Any        =						inputs["input_ids"][0].tolist()
            UpperCamelCase__    :     Tuple        =						time.time()
            UpperCamelCase__    :     Union[str, Any]        =						model.generate(inputs["input_ids"]    )
            UpperCamelCase__    :     Tuple        =						time.time()
            UpperCamelCase__    :     Optional[int]        =						outputs[0].tolist()
            UpperCamelCase__    :     List[Any]        =						len(lowercase__    ) if outputs[: len(lowercase__    )] != tokens else len(lowercase__    ) - len(lowercase__    )
            UpperCamelCase__    :     str        =						after_generate - before_generate
  
            UpperCamelCase__    :     List[str]        =						tokenizer.decode(lowercase__      ,					skip_special_tokens=lowercase__    )
            texts_outs.append(lowercase__    )
            generation_times.append(lowercase__    )
            gen_tokens.append(lowercase__    )
            print(f'Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'    )
  
       UpperCamelCase__    :     Optional[Any]        =						end_measure(lowercase__    )
       log_measures(lowercase__      ,					"Model generation"    )
  
       UpperCamelCase__    :     List[Any]        =						[gen / tok for gen, tok in zip(lowercase__      ,					lowercase__    )]
       UpperCamelCase__    :     List[str]        =						sum(lowercase__    ) / len(lowercase__    )
       print(f'Average time of generation per token: {avg_gen:.2f}s'    )
       print(f'First generation (avg time per token): {generation_times_per_token[0]:.2f}s'    )
       UpperCamelCase__    :     str        =						sum(generation_times_per_token[1:]    ) / (len(lowercase__    ) - 1)
       print(f'Average time of generation per token (excluding the first): {avg_gen:.2f}s'    )
  
  
  if __name__ == "__main__":
   main()
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)