


'''simple docstring'''




from typing import List

import datasets
from datasets.tasks import ImageClassification

from ..folder_based_builder import folder_based_builder


__lowercase  :      str      		=				datasets.utils.logging.get_logger(__name__)





class _snake_case   ( folder_based_builder.FolderBasedBuilderConfig   ):
							"""simple docstring"""





							__a												=     None
							__a												=     None





class _snake_case   ( folder_based_builder.FolderBasedBuilder   ):
							"""simple docstring"""





							__a												=     datasets.Image()
							__a												=     '''image'''
							__a												=     ImageFolderConfig
							__a												=     42  # definition at the bottom of the script
							__a												=     ImageClassification(image_column='''image'''    ,   label_column='''label'''   )

__lowercase  :      List[str]      		=				[
    """.blp""",
    """.bmp""",
    """.dib""",
    """.bufr""",
    """.cur""",
    """.pcx""",
    """.dcx""",
    """.dds""",
    """.ps""",
    """.eps""",
    """.fit""",
    """.fits""",
    """.fli""",
    """.flc""",
    """.ftc""",
    """.ftu""",
    """.gbr""",
    """.gif""",
    """.grib""",
    """.h5""",
    """.hdf""",
    """.png""",
    """.apng""",
    """.jp2""",
    """.j2k""",
    """.jpc""",
    """.jpf""",
    """.jpx""",
    """.j2c""",
    """.icns""",
    """.ico""",
    """.im""",
    """.iim""",
    """.tif""",
    """.tiff""",
    """.jfif""",
    """.jpe""",
    """.jpg""",
    """.jpeg""",
    """.mpg""",
    """.mpeg""",
    """.msp""",
    """.pcd""",
    """.pxr""",
    """.pbm""",
    """.pgm""",
    """.ppm""",
    """.pnm""",
    """.psd""",
    """.bw""",
    """.rgb""",
    """.rgba""",
    """.sgi""",
    """.ras""",
    """.tga""",
    """.icb""",
    """.vda""",
    """.vst""",
    """.webp""",
    """.wmf""",
    """.emf""",
    """.xbm""",
    """.xpm""",
]
__lowercase  :      List[str]      		=				IMAGE_EXTENSIONS









'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


__lowercase  :      str      		=				logging.get_logger(__name__)

__lowercase  :      Any      		=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase  :      Optional[Any]      		=				{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase  :      Tuple      		=				{
    """allenai/longformer-base-4096""": 40_96,
    """allenai/longformer-large-4096""": 40_96,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 40_96,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 40_96,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 40_96,
}





class _snake_case   ( a__   ):
							"""simple docstring"""





							__a												=     VOCAB_FILES_NAMES
							__a												=     PRETRAINED_VOCAB_FILES_MAP
							__a												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a												=     ['''input_ids''', '''attention_mask''']
							__a												=     LongformerTokenizer







							def __init__(   self			,				__A=None			,				__A=None			,				__A=None			,				__A="replace"			,				__A="<s>"			,				__A="</s>"			,				__A="</s>"			,				__A="<s>"			,				__A="<unk>"			,				__A="<pad>"			,				__A="<mask>"			,				__A=False			,				__A=True			,				**__A			,				)     ->Dict:



											"""simple docstring"""







											super().__init__(
											    __A			,				__A			,				tokenizer_file=__A			,				errors=__A			,				bos_token=__A			,				eos_token=__A			,				sep_token=__A			,				cls_token=__A			,				unk_token=__A			,				pad_token=__A			,				mask_token=__A			,				add_prefix_space=__A			,				trim_offsets=__A			,				**__A			,				)

											__lowerCamelCase		:      int											=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
											if pre_tok_state.get("add_prefix_space"			,				__A	) != add_prefix_space:
															__lowerCamelCase		:      Any											=					getattr(__A			,				pre_tok_state.pop("type"	)	)
															__lowerCamelCase		:      Any											=					add_prefix_space
															__lowerCamelCase		:      Union[str, Any]											=					pre_tok_class(**__A	)

											__lowerCamelCase		:      Any											=					add_prefix_space

											__lowerCamelCase		:      int											=					"post_processor"
											__lowerCamelCase		:      Optional[int]											=					getattr(self.backend_tokenizer			,				__A			,				__A	)
											if tokenizer_component_instance:
															__lowerCamelCase		:      Tuple											=					json.loads(tokenizer_component_instance.__getstate__()	)

															# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
															if "sep" in state:
																			__lowerCamelCase		:      Dict											=					tuple(state["sep"]	)
															if "cls" in state:
																			__lowerCamelCase		:      Union[str, Any]											=					tuple(state["cls"]	)

															__lowerCamelCase		:      Union[str, Any]											=					False

															if state.get("add_prefix_space"			,				__A	) != add_prefix_space:
																			__lowerCamelCase		:      Dict											=					add_prefix_space
																			__lowerCamelCase		:      str											=					True

															if state.get("trim_offsets"			,				__A	) != trim_offsets:
																			__lowerCamelCase		:      Tuple											=					trim_offsets
																			__lowerCamelCase		:      Optional[int]											=					True

															if changes_to_apply:
																			__lowerCamelCase		:      List[Any]											=					getattr(__A			,				state.pop("type"	)	)
																			__lowerCamelCase		:      int											=					component_class(**__A	)
																			setattr(self.backend_tokenizer			,				__A			,				__A	)







							@property
							def 		__SCREAMING_SNAKE_CASE(   self	)     ->str:



											"""simple docstring"""







											if self._mask_token is None:
															if self.verbose:
																			logger.error("Using mask_token, but it is not set yet."	)
															return None
											return str(self._mask_token	)







							@mask_token.setter
							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      List[Any]											=					AddedToken(__A			,				lstrip=__A			,				rstrip=__A	) if isinstance(__A			,				__A	) else value
											__lowerCamelCase		:      Optional[int]											=					value







							def 		__SCREAMING_SNAKE_CASE(   self			,				*__A			,				**__A	)     ->BatchEncoding:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[int]											=					kwargs.get("is_split_into_words"			,				__A	)
											assert self.add_prefix_space or not is_split_into_words, (
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__A			,				**__A	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				*__A			,				**__A	)     ->BatchEncoding:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					kwargs.get("is_split_into_words"			,				__A	)

											assert self.add_prefix_space or not is_split_into_words, (
											    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__A			,				**__A	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->Tuple[str]:



											"""simple docstring"""







											__lowerCamelCase		:      List[str]											=					self._tokenizer.model.save(__A			,				name=__A	)
											return tuple(__A	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A=None	)     ->Union[str, Any]:



											"""simple docstring"""







											__lowerCamelCase		:      Any											=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
															return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None	)     ->List[int]:



											"""simple docstring"""







											__lowerCamelCase		:      List[str]											=					[self.sep_token_id]
											__lowerCamelCase		:      Tuple											=					[self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep	) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]





