

"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__A						   =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A						   =			{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__A						   =			{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__A						   =			{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}





class     snake_case	(				__snake_case  ):
				SCREAMING_SNAKE_CASE_   :      List[str]         =  VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_   :      List[str]         =  PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_   :      Dict         =  PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_   :      Union[str, Any]         =  ElectraTokenizer

				def __init__(	self							:				Any     ,  UpperCamelCase__							:				Optional[Any]=None     ,  UpperCamelCase__							:				Dict=None     ,  UpperCamelCase__							:				Optional[Any]=True     ,  UpperCamelCase__							:				Any="[UNK]"     ,  UpperCamelCase__							:				str="[SEP]"     ,  UpperCamelCase__							:				int="[PAD]"     ,  UpperCamelCase__							:				Optional[Any]="[CLS]"     ,  UpperCamelCase__							:				int="[MASK]"     ,  UpperCamelCase__							:				Optional[int]=True     ,  UpperCamelCase__							:				List[Any]=None     ,  **UpperCamelCase__							:				Dict     ,  )->   int:



						'''simple docstring'''


						super().__init__(
						    UpperCamelCase__     ,  tokenizer_file=UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  sep_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  cls_token=UpperCamelCase__     ,  mask_token=UpperCamelCase__     ,  tokenize_chinese_chars=UpperCamelCase__     ,  strip_accents=UpperCamelCase__     ,  **UpperCamelCase__     ,  )

						__lowerCAmelCase:       str          =			json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    normalizer_state.get("lowercase"     ,  UpperCamelCase__) != do_lower_case
						    or normalizer_state.get("strip_accents"     ,  UpperCamelCase__) != strip_accents
						    or normalizer_state.get("handle_chinese_chars"     ,  UpperCamelCase__) != tokenize_chinese_chars
						):
								__lowerCAmelCase:       int          =			getattr(UpperCamelCase__     ,  normalizer_state.pop("type"))
								__lowerCAmelCase:       Dict          =			do_lower_case
								__lowerCAmelCase:       List[str]          =			strip_accents
								__lowerCAmelCase:       Any          =			tokenize_chinese_chars
								__lowerCAmelCase:       List[str]          =			normalizer_class(**UpperCamelCase__)

						__lowerCAmelCase:       Optional[Any]          =			do_lower_case

				def 							lowercase_     (	self							:				List[Any]     ,  UpperCamelCase__							:				int     ,  UpperCamelCase__							:				Dict=None)->   Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
								output += token_ids_a + [self.sep_token_id]

						return output

				def 							lowercase_     (	self							:				Union[str, Any]     ,  UpperCamelCase__							:				List[int]     ,  UpperCamelCase__							:				Optional[List[int]] = None)->   List[int]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			[self.sep_token_id]
						__lowerCAmelCase:       List[Any]          =			[self.cls_token_id]
						if token_ids_a is None:
								return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




				def 							lowercase_     (	self							:				int     ,  UpperCamelCase__							:				str     ,  UpperCamelCase__							:				Optional[str] = None)->   Tuple[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			self._tokenizer.model.save(UpperCamelCase__     ,  name=UpperCamelCase__)
						return tuple(UpperCamelCase__)





"""simple docstring"""


import pytest

from datasets import inspect_metric, list_metrics, load_metric






@pytest.fixture
def     a__    (    __SCREAMING_SNAKE_CASE )	->	Union[str, Any]:
		monkeypatch.setattr("datasets.utils.deprecation_utils._emitted_deprecation_warnings"					, set() )






@pytest.fixture
def     a__    (    __SCREAMING_SNAKE_CASE )	->	str:





		class     snake_case	:

						def __init__(	self							:				int     ,  UpperCamelCase__							:				Optional[int])->   Union[str, Any]:



								'''simple docstring'''


								__lowerCAmelCase:       str          =			metric_id





		class     snake_case	:
						SCREAMING_SNAKE_CASE_   :      List[Any]         =  [MetricMock(__snake_case  ) for metric_id in ["""accuracy""", """mse""", """precision""", """codeparrot/apps_metric"""]]

						def 							lowercase_     (	self							:				Tuple)->   Union[str, Any]:



								'''simple docstring'''


								return self._metrics


		monkeypatch.setattr("datasets.inspect.huggingface_hub"					, HfhMock() )






@pytest.mark.parametrize(
    "func, args"					, [(load_metric, ("metrics/mse",)), (list_metrics, ()), (inspect_metric, ("metrics/mse", "tmp_path"))] )
def     a__    (    __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE )	->	Optional[Any]:
		if "tmp_path" in args:
				__lowerCAmelCase:       Tuple          =			tuple(arg if arg != "tmp_path" else tmp_path for arg in args )
		with pytest.warns(__SCREAMING_SNAKE_CASE					, match="https://huggingface.co/docs/evaluate" ):
				func(*__SCREAMING_SNAKE_CASE )


