code: 
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import normalizers
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_splinter import SplinterTokenizer
  
  
  lowerCAmelCase__	:  Union[str, Any] 		=logging.get_logger(__name__)
  
  lowerCAmelCase__	:  Optional[Any] 		={'''vocab_file''': '''vocab.txt'''}
  
  lowerCAmelCase__	:  Any 		={
      '''vocab_file''': {
          '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
          '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
          '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
          '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
      }
  }
  
  lowerCAmelCase__	:  Tuple 		={
      '''tau/splinter-base''': 512,
      '''tau/splinter-base-qass''': 512,
      '''tau/splinter-large''': 512,
      '''tau/splinter-large-qass''': 512,
  }
  
  lowerCAmelCase__	:  Optional[int] 		={
      '''tau/splinter-base''': {'''do_lower_case''': False},
      '''tau/splinter-base-qass''': {'''do_lower_case''': False},
      '''tau/splinter-large''': {'''do_lower_case''': False},
      '''tau/splinter-large-qass''': {'''do_lower_case''': False},
  }
  
  class      UpperCAmelCase_						(     _SCREAMING_SNAKE_CASE ):
  
        '''simple docstring'''
  
  
  
  
        UpperCamelCase__    :				Tuple       						=						VOCAB_FILES_NAMES
        UpperCamelCase__    :				List[str]       						=						PRETRAINED_VOCAB_FILES_MAP
        UpperCamelCase__    :				Any       						=						PRETRAINED_INIT_CONFIGURATION
        UpperCamelCase__    :				Optional[Any]       						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        UpperCamelCase__    :				Optional[int]       						=						SplinterTokenizer
  
  
  
  
  
  
        def __init__(					self   ,							_A=None   ,							_A=None   ,							_A=True   ,							_A="[UNK]"   ,							_A="[SEP]"   ,							_A="[PAD]"   ,							_A="[CLS]"   ,							_A="[MASK]"   ,							_A="[QUESTION]"   ,							_A=True   ,							_A=None   ,							**_A   ,							):
  
  
  
  
  
  
  
               '''simple docstring'''
  
               super().__init__(
                   UpperCamelCase__   ,							tokenizer_file=UpperCamelCase__   ,							do_lower_case=UpperCamelCase__   ,							unk_token=UpperCamelCase__   ,							sep_token=UpperCamelCase__   ,							pad_token=UpperCamelCase__   ,							cls_token=UpperCamelCase__   ,							mask_token=UpperCamelCase__   ,							tokenize_chinese_chars=UpperCamelCase__   ,							strip_accents=UpperCamelCase__   ,							additional_special_tokens=(question_token,)   ,							**UpperCamelCase__   ,							)
  
               __SCREAMING_SNAKE_CASE								=      json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
               if (
                   pre_tok_state.get('lowercase'   ,							UpperCamelCase__	) != do_lower_case
                   or pre_tok_state.get('strip_accents'   ,							UpperCamelCase__	) != strip_accents
               ):
                      __SCREAMING_SNAKE_CASE								=      getattr(UpperCamelCase__   ,							pre_tok_state.pop('type'	)	)
                      __SCREAMING_SNAKE_CASE								=      do_lower_case
                      __SCREAMING_SNAKE_CASE								=      strip_accents
                      __SCREAMING_SNAKE_CASE								=      pre_tok_class(**UpperCamelCase__	)
  
               __SCREAMING_SNAKE_CASE								=      do_lower_case
  
  
  
  
  
  
        @property
        def  _A			(					self	):
  
  
  
  
  
  
  
               '''simple docstring'''
  
               return self.convert_tokens_to_ids(self.question_token	)
  
  
  
  
  
  
        def  _A			(					self   ,							_A   ,							_A = None	):
  
  
  
  
  
  
  
               '''simple docstring'''
  
               if token_ids_a is None:
                      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  
               __SCREAMING_SNAKE_CASE								=      [self.cls_token_id]
               __SCREAMING_SNAKE_CASE								=      [self.sep_token_id]
               __SCREAMING_SNAKE_CASE								=      [self.question_token_id] + [self.convert_tokens_to_ids('.'	)]
               if self.padding_side == "right":
                      # Input is question-then-context
                      return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
               else:
                      # Input is context-then-question
                      return cls + token_ids_a + sep + token_ids_a + question_suffix + sep
  
  
  
  
  
  
        def  _A			(					self   ,							_A   ,							_A = None	):
  
  
  
  
  
  
  
               '''simple docstring'''
  
               __SCREAMING_SNAKE_CASE								=      [self.sep_token_id]
               __SCREAMING_SNAKE_CASE								=      [self.cls_token_id]
               __SCREAMING_SNAKE_CASE								=      [self.question_token_id] + [self.convert_tokens_to_ids('.'	)]
               if token_ids_a is None:
                      return len(cls + token_ids_a + sep	) * [0]
  
               if self.padding_side == "right":
                      # Input is question-then-context
                      return len(cls + token_ids_a + question_suffix + sep	) * [0] + len(token_ids_a + sep	) * [1]
               else:
                      # Input is context-then-question
                      return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + question_suffix + sep	) * [1]
  
  
  
  
  
  
        def  _A			(					self   ,							_A   ,							_A = None	):
  
  
  
  
  
  
  
               '''simple docstring'''
  
               __SCREAMING_SNAKE_CASE								=      self._tokenizer.model.save(UpperCamelCase__   ,							name=UpperCamelCase__	)
               return tuple(UpperCamelCase__	)
  
  
  
  
  
  
  
style_context: 
  
  
  
  from __future__ import annotations
  
  from collections.abc import Iterable
  from dataclasses import dataclass
  from typing import Any
  
  class      UpperCAmelCase_						:
  
  						'''simple docstring'''
  
  
  
  
  						UpperCamelCase__    :				List[str]       						=						('''_front''', '''_back''', '''_len''')
  
  						@dataclass
  						class      UpperCAmelCase_						:
  
  												'''simple docstring'''
  
  
  
  
  												UpperCamelCase__    :				Any       						=						None
  												UpperCamelCase__    :				Deque._Node | None       						=						None
  												UpperCamelCase__    :				Deque._Node | None       						=						None
  
  
  
  
  
  
  
  						class      UpperCAmelCase_						:
  
  												'''simple docstring'''
  
  
  
  
  												UpperCamelCase__    :				Optional[int]       						=						('''_cur''',)
  
  
  
  
  
  
  												def __init__(					self   ,							_A	):
  
  
  
  
  
  
  
  																			'''simple docstring'''
  
  																			__SCREAMING_SNAKE_CASE								=      cur
  
  
  
  
  
  
  												def __iter__(					self	):
  
  
  
  
  
  
  
  																			'''simple docstring'''
  
  																			return self
  
  
  
  
  
  
  												def  _A			(					self	):
  
  
  
  
  
  
  
  																			'''simple docstring'''
  
  																			if self._cur is None:
  																										# finished iterating
  																										raise StopIteration
  																			__SCREAMING_SNAKE_CASE								=      self._cur.val
  																			__SCREAMING_SNAKE_CASE								=      self._cur.next_node
  
  																			return val
  
  
  
  
  
  
  
  						def __init__(					self   ,							_A = None	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													__SCREAMING_SNAKE_CASE								=      None
  													__SCREAMING_SNAKE_CASE								=      None
  													__SCREAMING_SNAKE_CASE								=      0
  
  													if iterable is not None:
  																				# append every value to the deque
  																				for val in iterable:
  																											self.append(_A	)
  
  
  
  
  
  
  						def  _A			(					self   ,							_A	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													__SCREAMING_SNAKE_CASE								=      self._Node(_A   ,							_A   ,							_A	)
  													if self.is_empty():
  																				# front = back
  																				__SCREAMING_SNAKE_CASE								=      __SCREAMING_SNAKE_CASE								=      node
  																				__SCREAMING_SNAKE_CASE								=      1
  													else:
  																				# connect nodes
  																				__SCREAMING_SNAKE_CASE								=      node
  																				__SCREAMING_SNAKE_CASE								=      self._back
  																				__SCREAMING_SNAKE_CASE								=      node  # assign new back to the new node
  
  																				self._len += 1
  
  																				# make sure there were no errors
  																				assert not self.is_empty(), "Error on appending value."
  
  
  
  
  
  
  						def  _A			(					self   ,							_A	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													__SCREAMING_SNAKE_CASE								=      self._Node(_A   ,							_A   ,							_A	)
  													if self.is_empty():
  																				# front = back
  																				__SCREAMING_SNAKE_CASE								=      __SCREAMING_SNAKE_CASE								=      node
  																				__SCREAMING_SNAKE_CASE								=      1
  													else:
  																				# connect nodes
  																				__SCREAMING_SNAKE_CASE								=      self._front
  																				__SCREAMING_SNAKE_CASE								=      node
  																				__SCREAMING_SNAKE_CASE								=      node  # assign new front to the new node
  
  																				self._len += 1
  
  																				# make sure there were no errors
  																				assert not self.is_empty(), "Error on appending value."
  
  
  
  
  
  
  						def  _A			(					self   ,							_A	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													for val in iterable:
  																				self.append(_A	)
  
  
  
  
  
  
  						def  _A			(					self   ,							_A	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													for val in iterable:
  																				self.appendleft(_A	)
  
  
  
  
  
  
  						def  _A			(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													assert not self.is_empty(), "Deque is empty."
  
  													__SCREAMING_SNAKE_CASE								=      self._back
  													__SCREAMING_SNAKE_CASE								=      self._back.prev_node  # set new back
  													# drop the last node - python will deallocate memory automatically
  													__SCREAMING_SNAKE_CASE								=      None
  
  													self._len -= 1
  
  													return topop.val
  
  
  
  
  
  
  						def  _A			(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													assert not self.is_empty(), "Deque is empty."
  
  													__SCREAMING_SNAKE_CASE								=      self._front
  													__SCREAMING_SNAKE_CASE								=      self._front.next_node  # set new front and drop the first node
  													__SCREAMING_SNAKE_CASE								=      None
  
  													self._len -= 1
  
  													return topop.val
  
  
  
  
  
  
  						def  _A			(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													return self._front is None
  
  
  
  
  
  
  						def __len__(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													return self._len
  
  
  
  
  
  
  						def __eq__(					self   ,							_A	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													if not isinstance(_A   ,							_A	):
  																				return NotImplemented
  
  													__SCREAMING_SNAKE_CASE								=      self._front
  													__SCREAMING_SNAKE_CASE								=      other._front
  
  													# if the length of the dequeues are not the same, they are not equal
  													if len(self	) != len(_A	):
  																				return False
  
  													while me is not None and oth is not None:
  																				# compare every value
  																				if me.val != oth.val:
  																											return False
  																				__SCREAMING_SNAKE_CASE								=      me.next_node
  																				__SCREAMING_SNAKE_CASE								=      oth.next_node
  
  													return True
  
  
  
  
  
  
  						def __iter__(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													return Deque._Iterator(self._front	)
  
  
  
  
  
  
  						def __repr__(					self	):
  
  
  
  
  
  
  
  													'''simple docstring'''
  
  													__SCREAMING_SNAKE_CASE								=      []
  													__SCREAMING_SNAKE_CASE								=      self._front
  													while aux is not None:
  																				# append the values in a list to display
  																				values_list.append(aux.val	)
  																				__SCREAMING_SNAKE_CASE								=      aux.next_node
  
  													return f"""[{", ".join(repr(_A	) for val in values_list	)}]"""
  
  
  if __name__ == "__main__":
  			import doctest
  
  			doctest.testmod()
  
  
  
  
  
  
  