





'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


lowerCAmelCase					:     Optional[int]												=logging.get_logger(__name__)


if is_vision_available():
 import PIL
class       a_   (				_lowerCAmelCase							):
     __A      =			["pixel_values"]





     def __init__( self		:					int	,      lowercase		:					bool = True	,      lowercase		:					Dict[str, int] = None	,      lowercase		:					PILImageResampling = PILImageResampling.BICUBIC	,      lowercase		:					bool = True	,      lowercase		:					Dict[str, int] = None	,      lowercase		:					bool = True	,      lowercase		:					Union[int, float] = 1 / 255	,      lowercase		:					bool = True	,      lowercase		:					Optional[Union[float, List[float]]] = None	,      lowercase		:					Optional[Union[float, List[float]]] = None	,      lowercase		:					bool = True	,      **lowercase		:					Dict	,      ):





        """simple docstring"""




        super().__init__(**lowercase						)
        lowercase_			:Optional[int]    			= size if size is not None else {"shortest_edge": 224}
        lowercase_			:Tuple    			= get_size_dict(lowercase	,      default_to_square=lowercase						)
        lowercase_			:Optional[int]    			= crop_size if crop_size is not None else {"height": 224, "width": 224}
        lowercase_			:List[Any]    			= get_size_dict(lowercase	,      default_to_square=lowercase	,      param_name="crop_size"						)

        lowercase_			:List[str]    			= do_resize
        lowercase_			:Union[str, Any]    			= size
        lowercase_			:List[Any]    			= resample
        lowercase_			:List[str]    			= do_center_crop
        lowercase_			:Union[str, Any]    			= crop_size
        lowercase_			:str    			= do_rescale
        lowercase_			:int    			= rescale_factor
        lowercase_			:Any    			= do_normalize
        lowercase_			:Any    			= image_mean if image_mean is not None else OPENAI_CLIP_MEAN
        lowercase_			:Optional[Any]    			= image_std if image_std is not None else OPENAI_CLIP_STD
        lowercase_			:Optional[int]    			= do_convert_rgb





     def 							lowercase__      ( self		:					List[str]	,      lowercase		:					np.ndarray	,      lowercase		:					Dict[str, int]	,      lowercase		:					PILImageResampling = PILImageResampling.BICUBIC	,      lowercase		:					Optional[Union[str, ChannelDimension]] = None	,      **lowercase		:					Optional[int]	,      ):





        """simple docstring"""




        lowercase_			:Optional[int]    			= get_size_dict(lowercase	,      default_to_square=lowercase						)
        if "shortest_edge" not in size:
           raise ValueError(F'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'						)
        lowercase_			:int    			= get_resize_output_image_size(lowercase	,      size=size["shortest_edge"]	,      default_to_square=lowercase						)
        return resize(lowercase	,      size=lowercase	,      resample=lowercase	,      data_format=lowercase	,      **lowercase						)





     def 							lowercase__      ( self		:					Union[str, Any]	,      lowercase		:					np.ndarray	,      lowercase		:					Dict[str, int]	,      lowercase		:					Optional[Union[str, ChannelDimension]] = None	,      **lowercase		:					str	,      ):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= get_size_dict(lowercase						)
        if "height" not in size or "width" not in size:
           raise ValueError(F'The `size` parameter must contain the keys (height, width). Got {size.keys()}'						)
        return center_crop(lowercase	,      size=(size["height"], size["width"])	,      data_format=lowercase	,      **lowercase						)





     def 							lowercase__      ( self		:					int	,      lowercase		:					np.ndarray	,      lowercase		:					Union[int, float]	,      lowercase		:					Optional[Union[str, ChannelDimension]] = None	,      **lowercase		:					Optional[Any]	,      ):





        """simple docstring"""




        return rescale(lowercase	,      scale=lowercase	,      data_format=lowercase	,      **lowercase						)





     def 							lowercase__      ( self		:					List[Any]	,      lowercase		:					np.ndarray	,      lowercase		:					Union[float, List[float]]	,      lowercase		:					Union[float, List[float]]	,      lowercase		:					Optional[Union[str, ChannelDimension]] = None	,      **lowercase		:					List[str]	,      ):





        """simple docstring"""




        return normalize(lowercase	,      mean=lowercase	,      std=lowercase	,      data_format=lowercase	,      **lowercase						)







     def 							lowercase__      ( self		:					List[str]	,      lowercase		:					ImageInput	,      lowercase		:					bool = None	,      lowercase		:					Dict[str, int] = None	,      lowercase		:					PILImageResampling = None	,      lowercase		:					bool = None	,      lowercase		:					int = None	,      lowercase		:					bool = None	,      lowercase		:					float = None	,      lowercase		:					bool = None	,      lowercase		:					Optional[Union[float, List[float]]] = None	,      lowercase		:					Optional[Union[float, List[float]]] = None	,      lowercase		:					bool = None	,      lowercase		:					Optional[Union[str, TensorType]] = None	,      lowercase		:					Optional[ChannelDimension] = ChannelDimension.FIRST	,      **lowercase		:					Dict	,      ):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= do_resize if do_resize is not None else self.do_resize
        lowercase_			:str    			= size if size is not None else self.size
        lowercase_			:int    			= get_size_dict(lowercase	,      param_name="size"	,      default_to_square=lowercase						)
        lowercase_			:List[str]    			= resample if resample is not None else self.resample
        lowercase_			:Dict    			= do_center_crop if do_center_crop is not None else self.do_center_crop
        lowercase_			:Optional[int]    			= crop_size if crop_size is not None else self.crop_size
        lowercase_			:str    			= get_size_dict(lowercase	,      param_name="crop_size"	,      default_to_square=lowercase						)
        lowercase_			:Optional[Any]    			= do_rescale if do_rescale is not None else self.do_rescale
        lowercase_			:Dict    			= rescale_factor if rescale_factor is not None else self.rescale_factor
        lowercase_			:Dict    			= do_normalize if do_normalize is not None else self.do_normalize
        lowercase_			:Optional[Any]    			= image_mean if image_mean is not None else self.image_mean
        lowercase_			:List[Any]    			= image_std if image_std is not None else self.image_std
        lowercase_			:List[str]    			= do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

        lowercase_			:List[str]    			= make_list_of_images(lowercase						)

        if not valid_images(lowercase						):
           raise ValueError(
               "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
               "torch.Tensor, tf.Tensor or jax.ndarray."						)

        if do_resize and size is None:
           raise ValueError("Size must be specified if do_resize is True."						)

        if do_center_crop and crop_size is None:
           raise ValueError("Crop size must be specified if do_center_crop is True."						)

        if do_rescale and rescale_factor is None:
           raise ValueError("Rescale factor must be specified if do_rescale is True."						)

        if do_normalize and (image_mean is None or image_std is None):
           raise ValueError("Image mean and std must be specified if do_normalize is True."						)

        # PIL RGBA images are converted to RGB
        if do_convert_rgb:
           lowercase_			:Any    			= [convert_to_rgb(lowercase						) for image in images]

        # All transformations expect numpy arrays.
        lowercase_			:List[Any]    			= [to_numpy_array(lowercase						) for image in images]

        if do_resize:
           lowercase_			:Optional[Any]    			= [self.resize(image=lowercase	,      size=lowercase	,      resample=lowercase						) for image in images]

        if do_center_crop:
           lowercase_			:Optional[Any]    			= [self.center_crop(image=lowercase	,      size=lowercase						) for image in images]

        if do_rescale:
           lowercase_			:Optional[int]    			= [self.rescale(image=lowercase	,      scale=lowercase						) for image in images]

        if do_normalize:
           lowercase_			:Any    			= [self.normalize(image=lowercase	,      mean=lowercase	,      std=lowercase						) for image in images]

        lowercase_			:Any    			= [to_channel_dimension_format(lowercase	,      lowercase						) for image in images]

        lowercase_			:Optional[Any]    			= {"pixel_values": images}
        return BatchFeature(data=lowercase	,      tensor_type=lowercase						)









'''simple docstring'''


import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class       a_   (				_lowerCAmelCase	,						unittest.TestCase							):
     __A      =			RobertaTokenizer
     __A      =			RobertaTokenizerFast
     __A      =			True
     __A      =			{"cls_token": "<s>"}





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        lowercase_			:List[Any]    			= [
            "l",
            "o",
            "w",
            "e",
            "r",
            "s",
            "t",
            "i",
            "d",
            "n",
            "\u0120",
            "\u0120l",
            "\u0120n",
            "\u0120lo",
            "\u0120low",
            "er",
            "\u0120lowest",
            "\u0120newer",
            "\u0120wider",
            "<unk>",
        ]
        lowercase_			:List[Any]    			= dict(zip(lowercase	,      range(len(lowercase						)						)						)						)
        lowercase_			:Optional[Any]    			= ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
        lowercase_			:Union[str, Any]    			= {"unk_token": "<unk>"}

        lowercase_			:Tuple    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["vocab_file"]						)
        lowercase_			:Union[str, Any]    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["merges_file"]						)
        with open(self.vocab_file	,      "w"	,      encoding="utf-8"						) as fp:
           fp.write(json.dumps(lowercase						) + "\n"						)
        with open(self.merges_file	,      "w"	,      encoding="utf-8"						) as fp:
           fp.write("\n".join(lowercase						)						)





     def 							lowercase__      ( self		:					str	,      **lowercase		:					List[str]						):





        """simple docstring"""




        kwargs.update(self.special_tokens_map						)
        return self.tokenizer_class.from_pretrained(self.tmpdirname	,      **lowercase						)





     def 							lowercase__      ( self		:					int	,      **lowercase		:					int						):





        """simple docstring"""




        kwargs.update(self.special_tokens_map						)
        return RobertaTokenizerFast.from_pretrained(self.tmpdirname	,      **lowercase						)





     def 							lowercase__      ( self		:					Optional[int]	,      lowercase		:					List[Any]						):





        """simple docstring"""




        lowercase_			:List[str]    			= "lower newer"
        lowercase_			:Any    			= "lower newer"
        return input_text, output_text





     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        lowercase_			:List[Any]    			= self.tokenizer_class(self.vocab_file	,      self.merges_file	,      **self.special_tokens_map						)
        lowercase_			:Dict    			= "lower newer"
        lowercase_			:Dict    			= ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
        lowercase_			:int    			= tokenizer.tokenize(lowercase						)  # , add_prefix_space=True)
        self.assertListEqual(lowercase	,      lowercase						)

        lowercase_			:Optional[Any]    			= tokens + [tokenizer.unk_token]
        lowercase_			:Any    			= [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase						)	,      lowercase						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        lowercase_			:Dict    			= self.get_tokenizer()

        self.assertListEqual(tokenizer.encode("Hello world!"	,      add_special_tokens=lowercase						)	,      [0, 31_414, 232, 328, 2]						)
        self.assertListEqual(
            tokenizer.encode("Hello world! cécé herlolip 418"	,      add_special_tokens=lowercase						)	,      [0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]	,      )





     @slow
     def 							lowercase__      ( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= self.tokenizer_class.from_pretrained("roberta-base"						)

        lowercase_			:Any    			= tokenizer.encode("sequence builders"	,      add_special_tokens=lowercase						)
        lowercase_			:str    			= tokenizer.encode("multi-sequence build"	,      add_special_tokens=lowercase						)

        lowercase_			:int    			= tokenizer.encode(
            "sequence builders"	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.encode(
            "sequence builders"	,      "multi-sequence build"	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)

        lowercase_			:str    			= tokenizer.build_inputs_with_special_tokens(lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.build_inputs_with_special_tokens(lowercase	,      lowercase						)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= self.get_tokenizer()

        lowercase_			:str    			= "Encode this sequence."
        lowercase_			:Tuple    			= tokenizer.byte_encoder[" ".encode("utf-8"						)[0]]

        # Testing encoder arguments
        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:List[Any]    			= tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)

        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase	,      add_prefix_space=lowercase						)
        lowercase_			:List[Any]    			= tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
        self.assertEqual(lowercase	,      lowercase						)

        tokenizer.add_special_tokens({"bos_token": "<s>"}						)
        lowercase_			:List[str]    			= tokenizer.encode(lowercase	,      add_special_tokens=lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.convert_ids_to_tokens(encoded[1]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)

        # Testing spaces after special tokens
        lowercase_			:Union[str, Any]    			= "<mask>"
        tokenizer.add_special_tokens(
            {"mask_token": AddedToken(lowercase	,      lstrip=lowercase	,      rstrip=lowercase						)}						)  # mask token has a left space
        lowercase_			:Any    			= tokenizer.convert_tokens_to_ids(lowercase						)

        lowercase_			:Tuple    			= "Encode <mask> sequence"
        lowercase_			:int    			= "Encode <mask>sequence"

        lowercase_			:str    			= tokenizer.encode(lowercase						)
        lowercase_			:Any    			= encoded.index(lowercase						)
        lowercase_			:Dict    			= tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
        self.assertEqual(lowercase	,      lowercase						)

        lowercase_			:str    			= tokenizer.encode(lowercase						)
        lowercase_			:int    			= encoded.index(lowercase						)
        lowercase_			:Optional[int]    			= tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
        self.assertNotEqual(lowercase	,      lowercase						)





     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        pass





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
              lowercase_			:List[Any]    			= self.rust_tokenizer_class.from_pretrained(lowercase	,      **lowercase						)
              lowercase_			:Dict    			= self.tokenizer_class.from_pretrained(lowercase	,      **lowercase						)
              lowercase_			:str    			= "A, <mask> AllenNLP sentence."
              lowercase_			:Tuple    			= tokenizer_r.encode_plus(lowercase	,      add_special_tokens=lowercase	,      return_token_type_ids=lowercase						)
              lowercase_			:str    			= tokenizer_p.encode_plus(lowercase	,      add_special_tokens=lowercase	,      return_token_type_ids=lowercase						)

              # token_type_ids should put 0 everywhere
              self.assertEqual(sum(tokens_r["token_type_ids"]						)	,      sum(tokens_p["token_type_ids"]						)						)

              # attention_mask should put 1 everywhere, so sum over length should be 1
              self.assertEqual(
                  sum(tokens_r["attention_mask"]						) / len(tokens_r["attention_mask"]						)	,      sum(tokens_p["attention_mask"]						) / len(tokens_p["attention_mask"]						)	,      )

              lowercase_			:Any    			= tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]						)
              lowercase_			:Union[str, Any]    			= tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]						)

              # Rust correctly handles the space before the mask while python doesnt
              self.assertSequenceEqual(tokens_p["input_ids"]	,      [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]						)
              self.assertSequenceEqual(tokens_r["input_ids"]	,      [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]						)

              self.assertSequenceEqual(
                  lowercase	,      ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)
              self.assertSequenceEqual(
                  lowercase	,      ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        for trim_offsets, add_prefix_space in itertools.product([True, False]	,      repeat=2						):
           lowercase_			:List[Any]    			= self.rust_tokenizer_class.from_pretrained(
               self.tmpdirname	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)

           lowercase_			:int    			= json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()						)
           lowercase_			:Any    			= json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()						)

           self.assertEqual(pre_tokenizer_state["add_prefix_space"]	,      lowercase						)

           self.assertEqual(post_processor_state["add_prefix_space"]	,      lowercase						)
           self.assertEqual(post_processor_state["trim_offsets"]	,      lowercase						)







     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'						):
              lowercase_			:Tuple    			= "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
              lowercase_			:Optional[Any]    			= F'{text_of_1_token} {text_of_1_token}'

              lowercase_			:int    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[int]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						) + 1, len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Dict    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:List[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						) + 1, len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Any    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						), len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:List[str]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:List[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (len(lowercase						), len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Dict    			= F' {text}'

              # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
              #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
              # )
              # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
              # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
              # self.assertEqual(
              #     encoding.offset_mapping[1],
              #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
              # )

              lowercase_			:Optional[Any]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Optional[Any]    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (1, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						) + 1, 1 + len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Union[str, Any]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Dict    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						), 1 + len(lowercase						) + 1 + len(lowercase						))	,      )

              lowercase_			:Optional[int]    			= self.rust_tokenizer_class.from_pretrained(
                  lowercase	,      use_fast=lowercase	,      add_prefix_space=lowercase	,      trim_offsets=lowercase						)
              lowercase_			:Dict    			= tokenizer_r(lowercase	,      return_offsets_mapping=lowercase	,      add_special_tokens=lowercase						)
              self.assertEqual(encoding.offset_mapping[0]	,      (0, 1 + len(lowercase						))						)
              self.assertEqual(
                  encoding.offset_mapping[1]	,      (1 + len(lowercase						), 1 + len(lowercase						) + 1 + len(lowercase						))	,      )


