


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase__   	=							logging.get_logger(__name__)

UpperCAmelCase__   	=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__   	=							{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__   	=							{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

UpperCAmelCase__   	=							{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}







class       lowerCamelCase__   (       lowerCAmelCase):
     SCREAMING_SNAKE_CASE__      			=			VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE__      			=			PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE__      			=			PRETRAINED_INIT_CONFIGURATION
     SCREAMING_SNAKE_CASE__      			=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE__      			=			RealmTokenizer





     def __init__(self		,     UpperCAmelCase=None		,     UpperCAmelCase=None		,     UpperCAmelCase=True		,     UpperCAmelCase="[UNK]"		,     UpperCAmelCase="[SEP]"		,     UpperCAmelCase="[PAD]"		,     UpperCAmelCase="[CLS]"		,     UpperCAmelCase="[MASK]"		,     UpperCAmelCase=True		,     UpperCAmelCase=None		,     **UpperCAmelCase		,     )     ->   Optional[Any]:
      super().__init__(
          UpperCAmelCase		,     tokenizer_file=UpperCAmelCase		,     do_lower_case=UpperCAmelCase		,     unk_token=UpperCAmelCase		,     sep_token=UpperCAmelCase		,     pad_token=UpperCAmelCase		,     cls_token=UpperCAmelCase		,     mask_token=UpperCAmelCase		,     tokenize_chinese_chars=UpperCAmelCase		,     strip_accents=UpperCAmelCase		,     **UpperCAmelCase		,     )

      _lowercase							      =json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          normalizer_state.get('''lowercase'''		,     UpperCAmelCase       ) != do_lower_case
          or normalizer_state.get('''strip_accents'''		,     UpperCAmelCase       ) != strip_accents
          or normalizer_state.get('''handle_chinese_chars'''		,     UpperCAmelCase       ) != tokenize_chinese_chars
      ):
       _lowercase							      =getattr(UpperCAmelCase		,     normalizer_state.pop('''type'''       )       )
       _lowercase							      =do_lower_case
       _lowercase							      =strip_accents
       _lowercase							      =tokenize_chinese_chars
       _lowercase							      =normalizer_class(**UpperCAmelCase       )

      _lowercase							      =do_lower_case





     def      __A					(self		,     UpperCAmelCase		,     **UpperCAmelCase       )     ->   Any:
      _lowercase							      =PaddingStrategy.MAX_LENGTH

      _lowercase							      =text
      _lowercase							      =kwargs.pop('''text_pair'''		,     UpperCAmelCase       )
      _lowercase							      =kwargs.pop('''return_tensors'''		,     UpperCAmelCase       )

      _lowercase							      ={
          '''input_ids''': [],
          '''attention_mask''': [],
          '''token_type_ids''': [],
      }

      for idx, candidate_text in enumerate(UpperCAmelCase       ):
       if batch_text_pair is not None:
        _lowercase							      =batch_text_pair[idx]
       else:
        _lowercase							      =None

       _lowercase							      =super().__call__(UpperCAmelCase		,     UpperCAmelCase		,     return_tensors=UpperCAmelCase		,     **UpperCAmelCase       )

       _lowercase							      =encoded_candidates.get('''input_ids'''       )
       _lowercase							      =encoded_candidates.get('''attention_mask'''       )
       _lowercase							      =encoded_candidates.get('''token_type_ids'''       )

       if encoded_input_ids is not None:
        output_data["input_ids"].append(UpperCAmelCase       )
       if encoded_attention_mask is not None:
        output_data["attention_mask"].append(UpperCAmelCase       )
       if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(UpperCAmelCase       )

      _lowercase							      ={key: item for key, item in output_data.items() if len(UpperCAmelCase       ) != 0}

      return BatchEncoding(UpperCAmelCase		,     tensor_type=UpperCAmelCase       )





     def      __A					(self		,     UpperCAmelCase		,     UpperCAmelCase=None       )     ->   Dict:
      _lowercase							      =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output





     def      __A					(self		,     UpperCAmelCase		,     UpperCAmelCase = None       )     ->   List[int]:
      _lowercase							      =[self.sep_token_id]
      _lowercase							      =[self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





     def      __A					(self		,     UpperCAmelCase		,     UpperCAmelCase = None       )     ->   Tuple[str]:
      _lowercase							      =self._tokenizer.model.save(UpperCAmelCase		,     name=UpperCAmelCase       )
      return tuple(UpperCAmelCase       )




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowercase_									=   {
    'configuration_bloom': ['BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BloomConfig', 'BloomOnnxConfig'],
}
try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   ['BloomTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   [
      'BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'BloomForCausalLM',
      'BloomModel',
      'BloomPreTrainedModel',
      'BloomForSequenceClassification',
      'BloomForTokenClassification',
      'BloomForQuestionAnswering',
  ]

if TYPE_CHECKING:
  from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_bloom_fast import BloomTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_bloom import (
        BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
        BloomForCausalLM,
        BloomForQuestionAnswering,
        BloomForSequenceClassification,
        BloomForTokenClassification,
        BloomModel,
        BloomPreTrainedModel,
    )

else:
  import sys

  lowercase_									=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


