code: 
  
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class     __a						(					A__ ,     unittest.TestCase ):
   _lowerCAmelCase : str 				=				GPTNeoXJapaneseTokenizer
   _lowerCAmelCase : Dict 				=				False
   _lowerCAmelCase : List[Any] 				=				{'''do_clean_text''': False, '''add_prefix_space''': False}
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().setUp()
  
        UpperCamelCase__    :     str        =						[
            "こん",
            "こんに",
            "にちは",
            "ばんは",
            "世界,㔺界",
            "、",
            "。",
            "<BR>",
            "<SP>",
            "<TAB>",
            "<URL>",
            "<EMAIL>",
            "<TEL>",
            "<DATE>",
            "<PRICE>",
            "<BLOCK>",
            "<KIGOU>",
            "<U2000U2BFF>",
            "<|emoji1|>",
            "<unk>",
            "<|startoftext|>",
            "<|endoftext|>",
        ]
        UpperCamelCase__    :     int        =						{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
        UpperCamelCase__    :     Dict        =						{"unk_token": "<unk>"}
  
        UpperCamelCase__    :     Dict        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["vocab_file"]		)
        UpperCamelCase__    :     List[Any]        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["emoji_file"]		)
        with open(self.vocab_file							,     "w"							,     encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
        with open(self.emoji_file							,     "w"		) as emoji_writer:
             emoji_writer.write(json.dumps(SCREAMING_SNAKE_CASE		)		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Any							,     **SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        kwargs.update(self.special_tokens_map		)
        return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Tuple							,     SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[str]        =						"こんにちは、世界。 \nこんばんは、㔺界。😀"
        UpperCamelCase__    :     Tuple        =						"こんにちは、世界。 \nこんばんは、世界。😀"
        return input_text, output_text
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Any							,     SCREAMING_SNAKE_CASE      :     Tuple		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__       ,  UpperCamelCase__    :     str        =						self.get_input_output_texts(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Any        =						tokenizer.encode(SCREAMING_SNAKE_CASE							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     str        =						tokenizer.decode(SCREAMING_SNAKE_CASE							,     clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE		)
        return text, ids
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Dict		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[int]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Dict        =						self.get_tokenizer()
  
        # Testing tokenization
        UpperCamelCase__    :     Union[str, Any]        =						"こんにちは、世界。　こんばんは、㔺界。"
        UpperCamelCase__    :     Optional[Any]        =						["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
        UpperCamelCase__    :     Union[str, Any]        =						tokenizer.tokenize(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        # Testing conversion to ids without special tokens
        UpperCamelCase__    :     str        =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
        UpperCamelCase__    :     Optional[int]        =						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        # Testing conversion to ids with special tokens
        UpperCamelCase__    :     Any        =						tokens + [tokenizer.unk_token]
        UpperCamelCase__    :     Union[str, Any]        =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
        UpperCamelCase__    :     List[str]        =						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   @slow
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[Any]        =						self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"		)
  
        UpperCamelCase__    :     Dict        =						tokenizer.encode("ありがとう。"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[str]        =						tokenizer.encode("どういたしまして。"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     int        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[int]        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        assert encoded_sentence == ids_a
        assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass
style_context: 
  
  
  
  
  
  
  import os
  from dataclasses import replace
  
  import jax
  import wandb
  from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
  from datasets import load_dataset
  from flax import jax_utils
  
  from transformers import BigBirdTokenizerFast
  
  
  if __name__ == "__main__":
   print('''#################### AVAILABLE DEVICES ####################''')
   print(jax.devices())
   print('''###########################################################''')
  
   # setup for wandb sweep
   lowerCamelCase			:							str       =Args()
   lowerCamelCase			:							str       =wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
   lowerCamelCase			:							Any       =dict(logger.config)
   del wandb_args["batch_size"]
   lowerCamelCase			:							str       =replace(args, **wandb_args)
   lowerCamelCase			:							List[str]       =args.base_dir + '''-''' + wandb.run.id
   lowerCamelCase			:							str       =replace(args, base_dir=base_dir)
   print(args)
  
   lowerCamelCase			:							List[Any]       =load_dataset('''json''', data_files=args.tr_data_path)['''train''']
   lowerCamelCase			:							str       =load_dataset('''json''', data_files=args.val_data_path)['''train''']
  
   # drop extra batch for now
   lowerCamelCase			:							List[Any]       =range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   lowerCamelCase			:							Union[str, Any]       =tr_dataset.shuffle().select(indices)
   lowerCamelCase			:							List[str]       =range(len(val_dataset) - len(val_dataset) % args.batch_size)
   lowerCamelCase			:							Dict       =val_dataset.shuffle().select(indices)
  
   if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
    lowerCamelCase			:							Optional[int]       =tr_dataset.shuffle().select(range(80000))
    lowerCamelCase			:							List[Any]       =val_dataset.shuffle().select(range(8000))
  
   print(tr_dataset)
   print(val_dataset)
  
   lowerCamelCase			:							int       =FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   lowerCamelCase			:							Union[str, Any]       =BigBirdTokenizerFast.from_pretrained(args.model_id)
   lowerCamelCase			:							Dict       =DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)
  
   lowerCamelCase			:							List[str]       ={
       '''lr''': args.lr,
       '''init_lr''': args.init_lr,
       '''warmup_steps''': args.warmup_steps,
       '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
       '''weight_decay''': args.weight_decay,
   }
   lowerCamelCase       ,    lowerCamelCase			:							Tuple       =build_tx(**tx_args)
  
   lowerCamelCase			:							Optional[Any]       =Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )
  
   lowerCamelCase			:							Tuple       =None
   lowerCamelCase			:							str       =trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
   try:
    trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
    print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')
  
   print('''SAVING WEIGHTS IN `final-weights`''')
   lowerCamelCase			:							str       =jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)