

import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class   __lowercase				( UpperCAmelCase_	):




			"""simple docstring"""







			def        _SCREAMING_SNAKE_CASE			( self  :     int):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.array(TypedSequence([1, 2, 3]))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([1, 2, 3])   ,						type=pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict):
									with self.assertRaises(lowerCAmelCase__):
															SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence([1, 2, 3]   ,						try_type=Value("bool")   ,						type=Value("int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence([1, 2, 3]   ,						type=Value("int32")))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
															SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						type=Value("int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([1, 2, 3]   ,						try_type=Value("int32")))
									self.assertEqual(arr.type   ,						pa.intaa())



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Dict  		=       pa.array(TypedSequence(["foo", "bar"]   ,						try_type=Value("int64")))
									self.assertEqual(arr.type   ,						pa.string())



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									SCREAMING_SNAKE_CASE_: int  		=       pa.array(TypedSequence([[[1, 2, 3]]]   ,						type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						ArrayaDExtensionType((1, 3)   ,						"int64"))



			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
															SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						type=ArrayaD((1, 3)   ,						"int64")))



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.array(TypedSequence([[[1, 2, 3]]]   ,						try_type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						ArrayaDExtensionType((1, 3)   ,						"int64"))



			def        _SCREAMING_SNAKE_CASE			( self  :     Any):
									SCREAMING_SNAKE_CASE_: List[Any]  		=       pa.array(TypedSequence(["foo", "bar"]   ,						try_type=ArrayaD((1, 3)   ,						"int64")))
									self.assertEqual(arr.type   ,						pa.string())




			@require_pil
			def        _SCREAMING_SNAKE_CASE			( self  :     str):
									import PIL.Image

									SCREAMING_SNAKE_CASE_: Tuple  		=       PIL.Image.fromarray(np.arange(10   ,						dtype=np.uinta).reshape(2   ,						5))
									with patch(
									    "datasets.arrow_writer.cast_to_python_objects"   ,						side_effect=lowerCAmelCase__) as mock_cast_to_python_objects:
															SCREAMING_SNAKE_CASE_: Tuple  		=       pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]   ,						type=Image()))
															SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[int]  		=       mock_cast_to_python_objects.call_args_list[-1]
															self.assertIn("optimize_list_casting"   ,						lowerCAmelCase__)
															self.assertFalse(kwargs["optimize_list_casting"])







def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Dict  		=       pa.BufferReader(_UpperCAmelCase    ) if isinstance(_UpperCAmelCase				,	pa.Buffer    ) else pa.memory_map(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.ipc.open_stream(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       f.read_all()
						assert len(pa_table.to_batches()    ) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: str  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: Optional[Any]  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: int  		=       Features({"labels": ClassLabel(names=["neg", "pos"]    )}    )
						with ArrowWriter(stream=_UpperCAmelCase				,	features=_UpperCAmelCase    ) as writer:
												writer.write({"labels": 0}    )
												writer.write({"labels": 1}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: Optional[int]  		=       pa.ipc.open_stream(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       f.read_all()
						SCREAMING_SNAKE_CASE_: str  		=       pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(_UpperCAmelCase    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: str  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												with pytest.raises(_UpperCAmelCase    ):
																		writer.write({"col_1": "foo", "col_2": 1}				,	key=[1, 2]    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												with pytest.raises(_UpperCAmelCase    ):
																		writer.write({"col_1": "foo", "col_2": 1}				,	key=10    )
																		writer.write({"col_1": "bar", "col_2": 2}				,	key=10    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Tuple  		=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]    )
def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase				,	hash_salt="split_name"				,	check_duplicates=_UpperCAmelCase				,	) as writer:
												writer.write({"col_1": "foo", "col_2": 1}				,	key=1    )
												writer.write({"col_1": "bar", "col_2": 2}				,	key=2    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: List[Any]  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )
												writer.write_batch({"col_1": [], "col_2": []}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: List[str]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: str  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Optional[Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: Dict  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]    )
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.BufferOutputStream()
						SCREAMING_SNAKE_CASE_: Any  		=       pa.schema(_UpperCAmelCase    ) if fields else None
						with ArrowWriter(stream=_UpperCAmelCase				,	schema=_UpperCAmelCase				,	writer_batch_size=_UpperCAmelCase    ) as writer:
												writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}    )    )
												writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}    )    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												SCREAMING_SNAKE_CASE_: List[str]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
						_check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )







def 			A_      (    ):
						with tempfile.TemporaryDirectory() as tmp_dir:
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       {"col_1": pa.string(), "col_2": pa.intaa()}
												SCREAMING_SNAKE_CASE_: Any  		=       os.path.join(_UpperCAmelCase				,	"test.arrow"    )
												with ArrowWriter(path=_UpperCAmelCase				,	schema=pa.schema(_UpperCAmelCase    )    ) as writer:
																		writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}    )
																		SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: List[Any]  		=       writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(_UpperCAmelCase				,	metadata=writer._schema.metadata    )
												_check_output(_UpperCAmelCase				,	1    )







def 			A_      (    _UpperCAmelCase    ):
						if pa.types.is_list(_UpperCAmelCase    ):
												return get_base_dtype(arr_type.value_type    )
						else:
												return arr_type







def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						if isinstance(lst[0]				,	_UpperCAmelCase    ):
												change_first_primitive_element_in_list(lst[0]				,	_UpperCAmelCase    )
						else:
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       value







@pytest.mark.parametrize("optimized_int_type, expected_dtype"				,	[(None, pa.intaa()), (Value("int32"    ), pa.intaa())]    )
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Tuple  		=       pa.array(TypedSequence(_UpperCAmelCase				,	optimized_int_type=_UpperCAmelCase    )    )
						assert get_base_dtype(arr.type    ) == expected_dtype







@pytest.mark.parametrize(
    "col, expected_dtype"				,	[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]				,	)
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase				,	_UpperCAmelCase    ):
						# in range
						SCREAMING_SNAKE_CASE_: List[str]  		=       pa.array(OptimizedTypedSequence(_UpperCAmelCase				,	col=_UpperCAmelCase    )    )
						assert get_base_dtype(arr.type    ) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												SCREAMING_SNAKE_CASE_: Optional[int]  		=       copy.deepcopy(_UpperCAmelCase    )
												SCREAMING_SNAKE_CASE_: str  		=       np.iinfo(expected_dtype.to_pandas_dtype()    ).max + 1
												change_first_primitive_element_in_list(_UpperCAmelCase				,	_UpperCAmelCase    )
												SCREAMING_SNAKE_CASE_: int  		=       pa.array(OptimizedTypedSequence(_UpperCAmelCase				,	col=_UpperCAmelCase    )    )
												assert get_base_dtype(arr.type    ) == pa.intaa()







@pytest.mark.parametrize("raise_exception"				,	[False, True]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: Any  		=       str(tmp_path / "dataset-train.arrow"    )
						try:
												with ArrowWriter(path=_UpperCAmelCase    ) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed







def 			A_      (    _UpperCAmelCase    ):
						SCREAMING_SNAKE_CASE_: str  		=       "mock://dataset-train.arrow"
						with ArrowWriter(path=_UpperCAmelCase				,	storage_options=mockfs.storage_options    ) as writer:
												assert isinstance(writer._fs				,	type(_UpperCAmelCase    )    )
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Any  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(_UpperCAmelCase    )







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: List[str]  		=       pa.BufferOutputStream()
						with ParquetWriter(stream=_UpperCAmelCase    ) as writer:
												writer.write({"col_1": "foo", "col_2": 1}    )
												writer.write({"col_1": "bar", "col_2": 2}    )
												SCREAMING_SNAKE_CASE_		,    SCREAMING_SNAKE_CASE_: Dict  		=       writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       pq.read_table(_UpperCAmelCase    )
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}







@require_pil
@pytest.mark.parametrize("embed_local_files"				,	[False, True]    )
def 			A_      (    _UpperCAmelCase				,	_UpperCAmelCase    ):
						import PIL.Image

						SCREAMING_SNAKE_CASE_: Any  		=       str(tmp_path / "test_image_rgb.jpg"    )
						PIL.Image.fromarray(np.zeros((5, 5)				,	dtype=np.uinta    )    ).save(_UpperCAmelCase				,	format="png"    )
						SCREAMING_SNAKE_CASE_: Any  		=       pa.BufferOutputStream()
						with ParquetWriter(
						    stream=_UpperCAmelCase				,	features=Features({"image": Image()}    )				,	embed_local_files=_UpperCAmelCase    ) as writer:
												writer.write({"image": image_path}    )
												writer.finalize()
						SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       pa.BufferReader(output.getvalue()    )
						SCREAMING_SNAKE_CASE_: pa.Table  		=       pq.read_table(_UpperCAmelCase    )
						SCREAMING_SNAKE_CASE_: int  		=       pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out["image"][0]["path"]				,	_UpperCAmelCase    )
												with open(_UpperCAmelCase				,	"rb"    ) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None







def 			A_      (    ):
						SCREAMING_SNAKE_CASE_: Dict  		=       pa.schema([pa.field("col_1"				,	pa.string()				,	nullable=_UpperCAmelCase    )]    )
						SCREAMING_SNAKE_CASE_: int  		=       pa.BufferOutputStream()
						with ArrowWriter(stream=_UpperCAmelCase    ) as writer:
												writer._build_writer(inferred_schema=_UpperCAmelCase    )
												assert writer._schema == pa.schema([pa.field("col_1"				,	pa.string()    )]    )











"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase : Optional[int]      						=						"▁"

_lowercase : Optional[Any]      						=						{"vocab_file": "spiece.model"}

_lowercase : Optional[Any]      						=						{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

_lowercase : Tuple      						=						{
    "google/pegasus-xsum": 5_1_2,
}


_lowercase : Optional[int]      						=						logging.get_logger(__name__)

class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					_a   			=							VOCAB_FILES_NAMES

					_a   			=							VOCAB_FILES_NAMES
					_a   			=							PRETRAINED_VOCAB_FILES_MAP
					_a   			=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_a   			=							['input_ids', 'attention_mask']



					def __init__(       self							:  Any,  lowerCamelCase							:  Optional[Any],  lowerCamelCase							:  Any="<pad>",  lowerCamelCase							:  Optional[Any]="</s>",  lowerCamelCase							:  Any="<unk>",  lowerCamelCase							:  Tuple="<mask_2>",  lowerCamelCase							:  int="<mask_1>",  lowerCamelCase							:  Optional[Any]=None,  lowerCamelCase							:  Dict=103,  lowerCamelCase							:  Optional[Dict[str, Any]] = None,  **lowerCamelCase							:  Optional[int],  )->					None:
								lowerCamelCase__		:		Union[str, Any]			=offset
								if additional_special_tokens is not None:
											if not isinstance(lowerCamelCase,  lowerCamelCase     ):
														raise TypeError(
														    F'''additional_special_tokens should be of type {type(lowerCamelCase     )}, but is'''
														    F''' {type(lowerCamelCase     )}'''     )

											lowerCamelCase__		:		Any			=(
											    ([mask_token_sent] + additional_special_tokens)
											    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
											    else additional_special_tokens
											)
											# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
											additional_special_tokens_extended += [
											    F'''<unk_{i}>''' for i in range(len(lowerCamelCase     ),  self.offset - 1     )
											]

											if len(set(lowerCamelCase     )     ) != len(lowerCamelCase     ):
														raise ValueError(
														    '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
														    F''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''     )
											lowerCamelCase__		:		Optional[Any]			=additional_special_tokens_extended
								else:
											lowerCamelCase__		:		Tuple			=[mask_token_sent] if mask_token_sent is not None else []
											additional_special_tokens += [F'''<unk_{i}>''' for i in range(2,  self.offset     )]

								lowerCamelCase__		:		Tuple			={} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    eos_token=lowerCamelCase,  unk_token=lowerCamelCase,  mask_token=lowerCamelCase,  pad_token=lowerCamelCase,  mask_token_sent=lowerCamelCase,  offset=lowerCamelCase,  additional_special_tokens=lowerCamelCase,  sp_model_kwargs=self.sp_model_kwargs,  **lowerCamelCase,  )
								lowerCamelCase__		:		Optional[int]			=mask_token_sent
								lowerCamelCase__		:		Optional[Any]			=vocab_file
								lowerCamelCase__		:		List[str]			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(lowerCamelCase     )

								# add special tokens to encoder dict
								lowerCamelCase__		:		Dict[int, str]			={
								    0: self.pad_token,
								    1: self.eos_token,
								}

								if self.mask_token_sent is not None:
											self.encoder.update(
											    {
											        2: self.mask_token_sent,
											        3: self.mask_token,
											    }     )

								if self.offset > 0:
											# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
											# mask_token_sent is already added to list -> so start at 1
											self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1,  self.offset - 1     )}     )

								lowerCamelCase__		:		Dict[str, int]			={v: k for k, v in self.encoder.items()}



					@property
					def 		snake_case       (       self							:  Union[str, Any]     )->					int:
								return len(self.sp_model     ) + self.offset



					def 		snake_case       (       self							:  Optional[Any]     )->					Dict[str, int]:
								lowerCamelCase__		:		List[Any]			={self.convert_ids_to_tokens(lowerCamelCase     ): i for i in range(self.vocab_size     )}
								vocab.update(self.added_tokens_encoder     )
								return vocab



					def __getstate__(       self							:  str     )->					List[Any]:
								lowerCamelCase__		:		Optional[Any]			=self.__dict__.copy()
								lowerCamelCase__		:		Optional[int]			=None
								return state



					def __setstate__(       self							:  Dict,  lowerCamelCase							:  int     )->					Optional[Any]:
								lowerCamelCase__		:		Optional[int]			=d

								# for backward compatibility
								if not hasattr(self,  '''sp_model_kwargs'''     ):
											lowerCamelCase__		:		str			={}

								lowerCamelCase__		:		Union[str, Any]			=spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(self.vocab_file     )



					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  str     )->					List[str]:
								return self.sp_model.encode(lowerCamelCase,  out_type=lowerCamelCase     )



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  str     )->					int:
								if token in self.decoder:
											return self.decoder[token]
								elif token in self.added_tokens_decoder:
											return self.added_tokens_decoder[token]
								lowerCamelCase__		:		Any			=self.sp_model.piece_to_id(lowerCamelCase     )
								return sp_id + self.offset



					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  int     )->					str:
								if index in self.encoder:
											return self.encoder[index]
								elif index in self.added_tokens_encoder:
											return self.added_tokens_encoder[index]
								else:
											lowerCamelCase__		:		Any			=self.sp_model.IdToPiece(index - self.offset     )
								return token



					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  Optional[int]     )->					Any:
								lowerCamelCase__		:		Optional[int]			=[]
								lowerCamelCase__		:		Tuple			=''''''
								for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
														out_string += self.sp_model.decode(lowerCamelCase     ) + token
														lowerCamelCase__		:		str			=[]
											else:
														current_sub_tokens.append(lowerCamelCase     )
								out_string += self.sp_model.decode(lowerCamelCase     )
								return out_string.strip()



					def 		snake_case       (       self							:  Union[str, Any],  lowerCamelCase							:  Union[str, Any]=False     )->					List[str]:
								return 1



					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  Optional[int]     )->					Tuple:
								lowerCamelCase__		:		Tuple			=set(self.all_special_ids     )  # call it once instead of inside list comp
								all_special_ids.remove(self.unk_token_id     )  # <unk> is only sometimes special

								return [1 if x in all_special_ids else 0 for x in seq]



					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  List,  lowerCamelCase							:  Optional[List] = None,  lowerCamelCase							:  bool = False     )->					List[int]:
								if already_has_special_tokens:
											return self._special_token_mask(lowerCamelCase     )
								elif token_ids_a is None:
											return self._special_token_mask(lowerCamelCase     ) + [1]
								else:
											return self._special_token_mask(token_ids_a + token_ids_a     ) + [1]



					def 		snake_case       (       self							:  Optional[Any],  lowerCamelCase							:  List[str],  lowerCamelCase							:  Optional[int]=None     )->					List[int]:
								if token_ids_a is None:
											return token_ids_a + [self.eos_token_id]
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return token_ids_a + token_ids_a + [self.eos_token_id]




					def 		snake_case       (       self							:  Optional[int],  lowerCamelCase							:  str,  lowerCamelCase							:  Optional[str] = None     )->					Tuple[str]:
								if not os.path.isdir(lowerCamelCase     ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
											return
								lowerCamelCase__		:		List[str]			=os.path.join(
								    lowerCamelCase,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

								if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
											copyfile(self.vocab_file,  lowerCamelCase     )
								elif not os.path.isfile(self.vocab_file     ):
											with open(lowerCamelCase,  '''wb'''     ) as fi:
														lowerCamelCase__		:		int			=self.sp_model.serialized_model_proto()
														fi.write(lowerCamelCase     )

								return (out_vocab_file,)




