






from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


A__	:			Dict			   = logging.get_logger(__name__)

A__	:			List[str]			   = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:			Dict			   = {
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

A__	:			Any			   = {'allegro/herbert-base-cased': 5_14}
A__	:			str			   = {}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       HerbertTokenizer




				def __init__(     self						:    Optional[Any],  lowerCamelCase						:    str=None,  lowerCamelCase						:    Tuple=None,  lowerCamelCase						:    Dict=None,  lowerCamelCase						:    Tuple="<s>",  lowerCamelCase						:    Dict="<unk>",  lowerCamelCase						:    Optional[Any]="<pad>",  lowerCamelCase						:    Union[str, Any]="<mask>",  lowerCamelCase						:    Tuple="</s>",  **lowerCamelCase						:    List[Any],  ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,  lowerCamelCase,  tokenizer_file=lowerCamelCase,  cls_token=lowerCamelCase,  unk_token=lowerCamelCase,  pad_token=lowerCamelCase,  mask_token=lowerCamelCase,  sep_token=lowerCamelCase,  **lowerCamelCase,  )




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.cls_token_id]
								lowercase__    =			[self.sep_token_id]
								if token_ids_a is None:
												return cls + token_ids_a + sep

								return cls + token_ids_a + sep + token_ids_a + sep




				def 				lowercase__   (     self						:    Optional[Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None,  lowerCamelCase						:    bool = False  ):

								'''simple docstring'''
								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=lowerCamelCase,  token_ids_a=lowerCamelCase,  already_has_special_tokens=lowerCamelCase  )

								if token_ids_a is None:
												return [1] + ([0] * len(lowerCamelCase  )) + [1]
								return [1] + ([0] * len(lowerCamelCase  )) + [1] + ([0] * len(lowerCamelCase  )) + [1]




				def 				lowercase__   (     self						:    Optional[int],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.sep_token_id]
								lowercase__    =			[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			self._tokenizer.model.save(lowerCamelCase,  name=lowerCamelCase  )
								return tuple(lowerCamelCase  )












from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				def __init__(     self						:    Dict,  lowerCamelCase						:    pyspark.sql.DataFrame,  lowerCamelCase						:    Optional[NamedSplit] = None,  lowerCamelCase						:    Optional[Features] = None,  lowerCamelCase						:    bool = True,  lowerCamelCase						:    str = None,  lowerCamelCase						:    bool = False,  lowerCamelCase						:    str = None,  lowerCamelCase						:    bool = True,  lowerCamelCase						:    str = "arrow",  **lowerCamelCase						:    str,  ):

								'''simple docstring'''
								super().__init__(
								    split=lowerCamelCase,  features=lowerCamelCase,  cache_dir=lowerCamelCase,  keep_in_memory=lowerCamelCase,  streaming=lowerCamelCase,  **lowerCamelCase,  )
								lowercase__    =			load_from_cache_file
								lowercase__    =			file_format
								lowercase__    =			Spark(
								    df=lowerCamelCase,  features=lowerCamelCase,  cache_dir=lowerCamelCase,  working_dir=lowerCamelCase,  **lowerCamelCase,  )




				def 				lowercase__   (     self						:    List[Any]  ):

								'''simple docstring'''
								if self.streaming:
												return self.builder.as_streaming_dataset(split=self.split  )
								lowercase__    =			None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
								self.builder.download_and_prepare(
								    download_mode=lowerCamelCase,  file_format=self._file_format,  )
								return self.builder.as_dataset(split=self.split  )




