



'''simple docstring'''

import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


__lowerCAmelCase									=   16
__lowerCAmelCase									=   32



def      __lowerCamelCase     (      lowerCAmelCase_						,			lowerCAmelCase_ = 16						,			lowerCAmelCase_ = "bert-base-cased"  )			->      str:
		_a     :      Optional[int]        =							AutoTokenizer.from_pretrained(lowerCAmelCase_  )
		_a     :      int        =							load_dataset('glue'						,			'mrpc'  )

		def tokenize_function(lowerCAmelCase_  ):
				# max_length=None => use the model max length (it's actually the default)
				_a     :      str        =							tokenizer(examples['sentence1']						,			examples['sentence2']						,			truncation=lowerCAmelCase_						,			max_length=lowerCAmelCase_  )
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		_a     :      str        =							datasets.map(
		    lowerCAmelCase_						,			batched=lowerCAmelCase_						,			remove_columns=['idx', 'sentence1', 'sentence2']						,			load_from_cache_file=lowerCAmelCase_  )

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		_a     :      int        =							tokenized_datasets.rename_column('label'						,			'labels'  )

		def collate_fn(lowerCAmelCase_  ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				if accelerator.distributed_type == DistributedType.TPU:
						return tokenizer.pad(lowerCAmelCase_						,			padding='max_length'						,			max_length=128						,			return_tensors='pt'  )
				return tokenizer.pad(lowerCAmelCase_						,			padding='longest'						,			return_tensors='pt'  )

		# Instantiate dataloaders.
		_a     :      Optional[int]        =							DataLoader(
		    tokenized_datasets['train']						,			shuffle=lowerCAmelCase_						,			collate_fn=lowerCAmelCase_						,			batch_size=lowerCAmelCase_  )
		_a     :      int        =							DataLoader(
		    tokenized_datasets['validation']						,			shuffle=lowerCAmelCase_						,			collate_fn=lowerCAmelCase_						,			batch_size=lowerCAmelCase_  )

		return train_dataloader, eval_dataloader



def      __lowerCamelCase     (      lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_  )			->      List[Any]:
		model.eval()
		_a     :      List[str]        =							0
		for step, batch in enumerate(lowerCAmelCase_  ):
				# We could avoid this line since we set the accelerator with `device_placement=True`.
				batch.to(accelerator.device  )
				with torch.no_grad():
						_a     :      int        =							model(**lowerCAmelCase_  )
				_a     :      Optional[int]        =							outputs.logits.argmax(dim=-1  )
				# It is slightly faster to call this once, than multiple times
				_a       ,					_a     :      List[str]        =							accelerator.gather(
				    (predictions, batch['labels'])  )  # If we are in a multiprocess environment, the last batch has duplicates
				if accelerator.use_distributed:
						if step == len(lowerCAmelCase_  ) - 1:
								_a     :      int        =							predictions[: len(eval_dataloader.dataset  ) - samples_seen]
								_a     :      int        =							references[: len(eval_dataloader.dataset  ) - samples_seen]
						else:
								samples_seen += references.shape[0]
				metric.add_batch(
				    predictions=lowerCAmelCase_						,			references=lowerCAmelCase_						,			)

		_a     :      List[Any]        =							metric.compute()
		return eval_metric["accuracy"]



def      __lowerCamelCase     (      lowerCAmelCase_						,			lowerCAmelCase_  )			->      Any:
		# Initialize accelerator
		_a     :      Optional[int]        =							Accelerator()

		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		_a     :      List[str]        =							config['lr']
		_a     :      int        =							int(config['num_epochs']  )
		_a     :      Union[str, Any]        =							int(config['seed']  )
		_a     :      Optional[Any]        =							int(config['batch_size']  )
		_a     :      List[str]        =							args.model_name_or_path

		set_seed(lowerCAmelCase_  )
		_a       ,					_a     :      Dict        =							get_dataloaders(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_  )

		# Instantiate the model (we build the model here so that the seed also control new weights initialization)
		_a     :      Tuple        =							AutoModelForSequenceClassification.from_pretrained(lowerCAmelCase_						,			return_dict=lowerCAmelCase_  )

		# Instantiate optimizer
		_a     :      List[str]        =							(
		    AdamW
		    if accelerator.state.deepspeed_plugin is None
		    or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
		    else DummyOptim
		)
		_a     :      Tuple        =							optimizer_cls(params=model.parameters()						,			lr=lowerCAmelCase_  )

		if accelerator.state.deepspeed_plugin is not None:
				_a     :      int        =							accelerator.state.deepspeed_plugin.deepspeed_config[
				    'gradient_accumulation_steps'
				]
		else:
				_a     :      Any        =							1
		_a     :      List[Any]        =							(len(lowerCAmelCase_  ) * num_epochs) // gradient_accumulation_steps

		# Instantiate scheduler
		if (
		    accelerator.state.deepspeed_plugin is None
		    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
		):
				_a     :      Any        =							get_linear_schedule_with_warmup(
				    optimizer=lowerCAmelCase_						,			num_warmup_steps=0						,			num_training_steps=lowerCAmelCase_						,			)
		else:
				_a     :      int        =							DummyScheduler(lowerCAmelCase_						,			total_num_steps=lowerCAmelCase_						,			warmup_num_steps=0  )

		# Prepare everything
		# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
		# prepare method.
		_a       ,					_a       ,					_a       ,					_a       ,					_a     :      Dict        =							accelerator.prepare(
		    lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_  )

		# We need to keep track of how many total steps we have iterated over
		_a     :      str        =							0
		# We also need to keep track of the stating epoch so files are named properly
		_a     :      Optional[Any]        =							0
		_a     :      List[Any]        =							evaluate.load('glue'						,			'mrpc'  )
		_a     :      List[Any]        =							num_epochs

		if args.partial_train_epoch is not None:
				_a     :      Dict        =							args.partial_train_epoch

		if args.resume_from_checkpoint:
				accelerator.load_state(args.resume_from_checkpoint  )
				_a     :      Any        =							args.resume_from_checkpoint.split('epoch_'  )[1]
				_a     :      Optional[int]        =							''
				for char in epoch_string:
						if char.isdigit():
								state_epoch_num += char
						else:
								break
				_a     :      Dict        =							int(lowerCAmelCase_  ) + 1
				_a     :      Dict        =							evaluation_loop(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_  )
				accelerator.print('resumed checkpoint performance:'						,			lowerCAmelCase_  )
				accelerator.print('resumed checkpoint\'s scheduler\'s lr:'						,			lr_scheduler.get_lr()[0]  )
				accelerator.print('resumed optimizers\'s lr:'						,			optimizer.param_groups[0]['lr']  )
				with open(os.path.join(args.output_dir						,			f"""state_{starting_epoch-1}.json"""  )						,			'r'  ) as f:
						_a     :      str        =							json.load(lowerCAmelCase_  )
						assert resumed_state["accuracy"] == accuracy, "Accuracy mismatch, loading from checkpoint failed"
						assert (
						    resumed_state["lr"] == lr_scheduler.get_lr()[0]
						), "Scheduler learning rate mismatch, loading from checkpoint failed"
						assert (
						    resumed_state["optimizer_lr"] == optimizer.param_groups[0]["lr"]
						), "Optimizer learning rate mismatch, loading from checkpoint failed"
						assert resumed_state["epoch"] == starting_epoch - 1, "Epoch mismatch, loading from checkpoint failed"
						return

    # Now we train the model
		_a     :      Any        =							{}
		for epoch in range(lowerCAmelCase_						,			lowerCAmelCase_  ):
				model.train()
				for step, batch in enumerate(lowerCAmelCase_  ):
						_a     :      Optional[int]        =							model(**lowerCAmelCase_  )
						_a     :      str        =							outputs.loss
						_a     :      str        =							loss / gradient_accumulation_steps
						accelerator.backward(lowerCAmelCase_  )
						if step % gradient_accumulation_steps == 0:
								optimizer.step()
								lr_scheduler.step()
								optimizer.zero_grad()

						overall_step += 1
				_a     :      List[Any]        =							f"""epoch_{epoch}"""
				_a     :      Union[str, Any]        =							os.path.join(args.output_dir						,			lowerCAmelCase_  )
				accelerator.save_state(lowerCAmelCase_  )
				_a     :      List[str]        =							evaluation_loop(lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_						,			lowerCAmelCase_  )
				_a     :      Optional[Any]        =							accuracy
				_a     :      List[Any]        =							lr_scheduler.get_lr()[0]
				_a     :      Any        =							optimizer.param_groups[0]['lr']
				_a     :      Dict        =							epoch
				_a     :      List[Any]        =							overall_step
				accelerator.print(f"""epoch {epoch}:"""						,			lowerCAmelCase_  )

				accelerator.wait_for_everyone()
				if accelerator.is_main_process:
						with open(os.path.join(args.output_dir						,			f"""state_{epoch}.json"""  )						,			'w'  ) as f:
								json.dump(lowerCAmelCase_						,			lowerCAmelCase_  )



def      __lowerCamelCase     (      )			->      List[Any]:
		_a     :      Union[str, Any]        =							argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.'  )
		parser.add_argument(
		    '--model_name_or_path'						,			type=lowerCAmelCase_						,			default='bert-base-cased'						,			help='Path to pretrained model or model identifier from huggingface.co/models.'						,			required=lowerCAmelCase_						,			)
		parser.add_argument(
		    '--output_dir'						,			type=lowerCAmelCase_						,			default='.'						,			help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'						,			)
		parser.add_argument(
		    '--resume_from_checkpoint'						,			type=lowerCAmelCase_						,			default=lowerCAmelCase_						,			help='If the training should continue from a checkpoint folder.'						,			)
		parser.add_argument(
		    '--partial_train_epoch'						,			type=lowerCAmelCase_						,			default=lowerCAmelCase_						,			help='If passed, the training will stop after this number of epochs.'						,			)
		parser.add_argument(
		    '--num_epochs'						,			type=lowerCAmelCase_						,			default=2						,			help='Number of train epochs.'						,			)
		_a     :      Tuple        =							parser.parse_args()
		_a     :      Optional[Any]        =							{'lr': 2E-5, 'num_epochs': args.num_epochs, 'seed': 42, 'batch_size': 16}

		training_function(lowerCAmelCase_						,			lowerCAmelCase_  )


if __name__ == "__main__":
							main()











'''simple docstring'''


import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class       a_   (				_lowerCAmelCase	,						unittest.TestCase							):
     __A      =			RobertaTokenizer
     __A      =			RobertaTokenizerFast
     __A      =			True
     __A      =			{"cls_token": "<s>"}





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        lowercase_			:List[Any]    			= [
            "l",
            "o",
            "w",
            "e",
            "r",
            "s",
            "t",
            "i",
            "d",
            "n",
            "\u0120",
            "\u0120l",
            "\u0120n",
            "\u0120lo",
            "\u0120low",
            "er",
            "\u0120lowest",
            "\u0120newer",
            "\u0120wider",
            "<unk>",
        ]
        lowercase_			:List[Any]    			= dict(zip(lowercase	,      range(len(lowercase						)						)						)						)
        lowercase_			:Optional[Any]    			= ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
        lowercase_			:Union[str, Any]    			= {"unk_token": "<unk>"}

        lowercase_			:Tuple    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["vocab_file"]						)
        lowercase_			:Union[str, Any]    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["merges_file"]						)
        with open(self.vocab_file	,      "w"	,      encoding="utf-8"						) as fp:
           fp.write(json.dumps(lowercase						) + "\n"						)
        with open(self.merges_file	,      "w"	,      encoding="utf-8"						) as fp:
           fp.write("\n".join(lowercase						)						)





     def 							lowercase__      ( self		:					str	,      **lowercase		:					List[str]						):





        """simple docstring"""




        kwargs.update(self.special_tokens_map						)
        return self.tokenizer_class.from_pretrained(self.tmpdirname	,      **lowercase						)





     def 							lowercase__      ( self		:					int	,      **lowercase		:					int						):





        """simple docstring"""




        kwargs.update(self.special_tokens_map						)
        return RobertaTokenizerFast.from_pretrained(self.tmpdirname	,      **lowercase						)





     def 							lowercase__      ( self		:					Optional[int]	,      lowercase		:					List[Any]						):





        """simple docstring"""




        lowercase_			:List[str]    			= "lower newer"
        lowercase_			:Any    			= "lower newer"
        return input_text, output_text





     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        lowercase_			:List[Any]    			= self.tokenizer_class(self.vocab_file	,      self.merges_file	,      **self.special_tokens_map						)
        lowercase_			:Dict    			= "lower newer"
        lowercase_			:Dict    			= ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
        lowercase_			:int    			= tokenizer.tokenize(lowercase						)  # , add_prefix_space=True)
        self.assertListEqual(lowercase	,      lowercase						)

        lowercase_			:Optional[Any]    			= tokens + [tokenizer.unk_token]
        lowercase_			:Any    			= [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase						)	,      lowercase						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        lowercase_			:Dict    			= self.get_tokenizer()

        self.assertListEqual(tokenizer.encode("Hello world!"	,      add_special_tokens=lowercase						)	,      [0, 31_414, 232, 328, 2]						)
        self.assertListEqual(
            tokenizer.encode("Hello world! cécé herlolip 418"	,      add_special_tokens=lowercase						)	,      [0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]	,      )





     @slow
     def 							lowercase__      ( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= self.tokenizer_class.from_pretrained("roberta-base"						)

        lowercase_			:Any    			= tokenizer.encode("sequence builders"	,      add_special_tokens=lowercase						)
        lowercase_			:str    			= tokenizer.encode("multi-sequence build"	,      add_special_tokens=lowercase						)

        lowercase_			:int    			= tokenizer.encode(
            "sequence builders"	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.encode(
            "sequence builders"	,      "multi-sequence build"	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)

        lowercase_			:str    			= tokenizer.build_inputs_with_special_tokens(lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.build_inputs_with_special_tokens(lowercase	,      lowercase						)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= self.get_tokenizer()

        lowercase_			:str    			= "Encode this sequence."
        lowercase_			:Tuple    			= tokenizer.byte_encoder[" ".encode("utf-8"						)[0]]

        # Testing encoder arguments
        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:List[Any]    			= tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)

        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:List[Any]    			= tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
        self.assertEqual(lowercase	,      lowercase						)

        tokenizer.add_special_tokens({"bos_token": "<s>"}						)
        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.convert_ids_to_tokens(encoded[1]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)

        # Testing spaces after special tokens
        lowercase_			:Union[str, Any]    			= "<mask>"
        tokenizer.add_special_tokens(
            {"mask_token": AddedToken(lowercase	,      lstrip=lowercase	,      rstrip=lowercase						)}						)  # mask token has a left space
        lowercase_			:Any    			= tokenizer.convert_tokens_to_ids(lowercase						)

        lowercase_			:Tuple    			= "Encode <mask> sequence"
        lowercase_			:int    			= "Encode <mask>sequence"

        lowercase_			:str    			= tokenizer.encode(lowercase						)
        lowercase_			:Any    			= encoded.index(lowercase						)
        lowercase_			:Dict    			= tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
        self.assertEqual(lowercase	,      lowercase						)

        lowercase_			:str    			= tokenizer.encode(lowercase						)
        lowercase_			:int    			= encoded.index(lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)





     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        pass





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
              lowercase_			:List[Any]    			= self.rust_tokenizer_class.from_pretrained(lowercase	,      **lowercase						)
              lowercase_			:Dict    			= self.tokenizer_class.from_pretrained(lowercase	,      **lowercase						)
              lowercase_			:str    			= "A, <mask> AllenNLP sentence."
              lowercase_			:Tuple    			= tokenizer_r.encode_plus(lowercase	,      add_special_tokens=lowercase	,      return_token_type_ids=lowercase						)
              lowercase_			:str    			= tokenizer_p.encode_plus(lowercase	,      add_special_tokens=lowercase	,      return_token_type_ids=lowercase						)

              # token_type_ids should put 0 everywhere
              self.assertEqual(sum(tokens_r["token_type_ids"]						)	,      sum(tokens_p["token_type_ids"]						)						)

              # attention_mask should put 1 everywhere, so sum over length should be 1
              self.assertEqual(
                  sum(tokens_r["attention_mask"]						) / len(tokens_r["attention_mask"]						)	,      sum(tokens_p["attention_mask"]						) / len(tokens_p["attention_mask"]						)	,      )

              lowercase_			:Any    			= tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]						)
              lowercase_			:Union[str, Any]    			= tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]						)

              # Rust correctly handles the space before the mask while python doesnt
              self.assertSequenceEqual(tokens_p["input_ids"]	,      [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]						)
              self.assertSequenceEqual(tokens_r["input_ids"]	,      [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]						)

              self.assertSequenceEqual(
                  lowercase	,      ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)
              self.assertSequenceEqual(
                  lowercase	,      ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        for trim_offsets, add_prefix_space in itertools.product([True, False]	,      repeat=2						):
           lowercase_			:List[Any]    			= self.rust_tokenizer_class.from_pretrained(
               self.tmpdirname	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)

           lowercase_			:int    			= json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()						)
           lowercase_			:Any    			= json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()						)

           self.assertEqual(pre_tokenizer_state["add_prefix_space"]	,      lowercase						)

           self.assertEqual(post_processor_state["add_prefix_space"]	,      lowercase						)
           self.assertEqual(post_processor_state["trim_offsets"]	,      lowercase						)







     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
              lowercase_			:Tuple    			= "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
              lowercase_			:Optional[Any]    			= F'{text_of_1_token} {text_of_1_token}'

              lowercase_			:int    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[int]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						) + 1, len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Dict    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:List[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						) + 1, len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Any    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						), len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:List[str]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:List[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						), len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Dict    			= F' {text}'

              # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
              #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
              # )
              # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
              # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
              # self.assertEqual(
              #     encoding.offset_mapping[1],
              #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
              # )

              lowercase_			:Optional[Any]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (1, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						) + 1, 1 + len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Union[str, Any]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Dict    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						), 1 + len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Optional[int]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Dict    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						), 1 + len(lowercase						) + 1 + len(lowercase						))	,      )


