



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase__   =       logging.get_logger(__name__)

UpperCAmelCase__   =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__   =       {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__   =       {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

UpperCAmelCase__   =       {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}







class 			__lowerCAmelCase    (			A						):
			UpperCamelCase   					=     VOCAB_FILES_NAMES
			UpperCamelCase   					=     PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase   					=     PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase   					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase   					=     RealmTokenizer



			def __init__(						self   :  Optional[int] ,		A   :  Tuple=None ,		A   :  str=None ,		A   :  Tuple=True ,		A   :  Dict="[UNK]" ,		A   :  Any="[SEP]" ,		A   :  int="[PAD]" ,		A   :  List[str]="[CLS]" ,		A   :  Optional[int]="[MASK]" ,		A   :  Optional[Any]=True ,		A   :  Optional[Any]=None ,		**A   :  Any ,		)   ->    Optional[int]:



						"""simple docstring"""






						super().__init__(
						    A ,		tokenizer_file=A ,		do_lower_case=A ,		unk_token=A ,		sep_token=A ,		pad_token=A ,		cls_token=A ,		mask_token=A ,		tokenize_chinese_chars=A ,		strip_accents=A ,		**A ,		)

						_UpperCAmelCase		       =   json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    normalizer_state.get('lowercase' ,		A) != do_lower_case
						    or normalizer_state.get('strip_accents' ,		A) != strip_accents
						    or normalizer_state.get('handle_chinese_chars' ,		A) != tokenize_chinese_chars
						):
									_UpperCAmelCase		       =   getattr(A ,		normalizer_state.pop('type'))
									_UpperCAmelCase		       =   do_lower_case
									_UpperCAmelCase		       =   strip_accents
									_UpperCAmelCase		       =   tokenize_chinese_chars
									_UpperCAmelCase		       =   normalizer_class(**A)

						_UpperCAmelCase		       =   do_lower_case



			def     _lowerCamelCase       (						self   :  Tuple ,		A   :  Dict ,		**A   :  Tuple)   ->    Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase		       =   PaddingStrategy.MAX_LENGTH

						_UpperCAmelCase		       =   text
						_UpperCAmelCase		       =   kwargs.pop('text_pair' ,		A)
						_UpperCAmelCase		       =   kwargs.pop('return_tensors' ,		A)

						_UpperCAmelCase		       =   {
						    'input_ids': [],
						    'attention_mask': [],
						    'token_type_ids': [],
						}

						for idx, candidate_text in enumerate(A):
									if batch_text_pair is not None:
												_UpperCAmelCase		       =   batch_text_pair[idx]
									else:
												_UpperCAmelCase		       =   None

									_UpperCAmelCase		       =   super().__call__(A ,		A ,		return_tensors=A ,		**A)

									_UpperCAmelCase		       =   encoded_candidates.get('input_ids')
									_UpperCAmelCase		       =   encoded_candidates.get('attention_mask')
									_UpperCAmelCase		       =   encoded_candidates.get('token_type_ids')

									if encoded_input_ids is not None:
												output_data["input_ids"].append(A)
									if encoded_attention_mask is not None:
												output_data["attention_mask"].append(A)
									if encoded_token_type_ids is not None:
												output_data["token_type_ids"].append(A)

						_UpperCAmelCase		       =   {key: item for key, item in output_data.items() if len(A) != 0}

						return BatchEncoding(A ,		tensor_type=A)



			def     _lowerCamelCase       (						self   :  Tuple ,		A   :  List[Any] ,		A   :  List[str]=None)   ->    str:



						"""simple docstring"""






						_UpperCAmelCase		       =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output



			def     _lowerCamelCase       (						self   :  List[Any] ,		A   :  int ,		A   :  Optional[Any] = None)   ->    List[int]:



						"""simple docstring"""






						_UpperCAmelCase		       =   [self.sep_token_id]
						_UpperCAmelCase		       =   [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



			def     _lowerCamelCase       (						self   :  str ,		A   :  int ,		A   :  List[Any] = None)   ->    Tuple[str]:



						"""simple docstring"""






						_UpperCAmelCase		       =   self._tokenizer.model.save(A ,		name=A)
						return tuple(A)



from .imports import is_tqdm_available


if is_tqdm_available():
  from tqdm.auto import tqdm as _tqdm

from ..state import PartialState
def      a      (			A__	:   bool = True      ,   *A__	:   int      ,   **A__	:   Union[str, Any]     )    ->      List[str]:





      """simple docstring"""



      if not is_tqdm_available():
            raise ImportError('Accelerate\'s `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`.'     )
      _lowercase	       =False
      if main_process_only:
            _lowercase	       =PartialState().local_process_index == 0
      return _tqdm(*A__      ,   **A__      ,   disable=A__     )


