



import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        __snake_case		(  UpperCamelCase_		,unittest.TestCase		):
       _a       =				HerbertTokenizer
       _a       =				HerbertTokenizerFast
       _a       =				True




       def      UpperCAmelCase__    ( self  : Dict):
         super().setUp()

         # Use a simpler test file without japanese/chinese characters
         with open(F"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"""       ,       encoding='''utf-8''') as f_data:
           lowerCAmelCase_    :  Optional[int]             =				f_data.read().replace('''\n\n'''       ,       '''\n''').strip()

         lowerCAmelCase_    :  Dict             =				[
             '''<s>''',
             '''</s>''',
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''w</w>''',
             '''r</w>''',
             '''t</w>''',
             '''lo''',
             '''low''',
             '''er</w>''',
             '''low</w>''',
             '''lowest</w>''',
             '''newer</w>''',
             '''wider</w>''',
             ''',</w>''',
             '''<unk>''',
         ]
         lowerCAmelCase_    :  Union[str, Any]             =				dict(zip(A_       ,       range(len(A_))))
         lowerCAmelCase_    :  Any             =				['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

         lowerCAmelCase_    :  List[Any]             =				os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''vocab_file'''])
         lowerCAmelCase_    :  Tuple             =				os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''merges_file'''])
         with open(self.vocab_file       ,       '''w''') as fp:
           fp.write(json.dumps(A_))
         with open(self.merges_file       ,       '''w''') as fp:
           fp.write('''\n'''.join(A_))




       def      UpperCAmelCase__    ( self  : Any       ,       A_  : Tuple):
         lowerCAmelCase_    :  Dict             =				'''lower newer'''
         lowerCAmelCase_    :  List[str]             =				'''lower newer'''
         return input_text, output_text




       def      UpperCAmelCase__    ( self  : Optional[Any]):
         lowerCAmelCase_    :  List[Any]             =				self.tokenizer_class(vocab_file=self.vocab_file       ,       merges_file=self.merges_file)

         lowerCAmelCase_    :  Tuple             =				'''lower'''
         lowerCAmelCase_    :  str             =				['''low''', '''er</w>''']
         lowerCAmelCase_    :  Optional[Any]             =				tokenizer.tokenize(A_)
         self.assertListEqual(A_       ,       A_)

         lowerCAmelCase_    :  str             =				tokens + ['''<unk>''']
         lowerCAmelCase_    :  Dict             =				[1_6, 1_7, 2_3]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(A_)       ,       A_)




       def      UpperCAmelCase__    ( self  : str):
         if not self.test_rust_tokenizer:
           return

         lowerCAmelCase_    :  Optional[Any]             =				self.get_tokenizer()
         lowerCAmelCase_    :  Union[str, Any]             =				self.get_rust_tokenizer()

         lowerCAmelCase_    :  Any             =				'''lower,newer'''

         lowerCAmelCase_    :  Optional[Any]             =				tokenizer.tokenize(A_)
         lowerCAmelCase_    :  List[str]             =				rust_tokenizer.tokenize(A_)
         self.assertListEqual(A_       ,       A_)

         lowerCAmelCase_    :  Dict             =				tokenizer.encode(A_       ,       add_special_tokens=A_)
         lowerCAmelCase_    :  Optional[int]             =				rust_tokenizer.encode(A_       ,       add_special_tokens=A_)
         self.assertListEqual(A_       ,       A_)

         lowerCAmelCase_    :  List[Any]             =				self.get_rust_tokenizer()
         lowerCAmelCase_    :  Tuple             =				tokenizer.encode(A_)
         lowerCAmelCase_    :  str             =				rust_tokenizer.encode(A_)
         self.assertListEqual(A_       ,       A_)







       @slow
       def      UpperCAmelCase__    ( self  : Any):
         lowerCAmelCase_    :  List[Any]             =				self.tokenizer_class.from_pretrained('''allegro/herbert-base-cased''')

         lowerCAmelCase_    :  str             =				tokenizer.encode('''konstruowanie sekwencji'''       ,       add_special_tokens=A_)
         lowerCAmelCase_    :  Optional[Any]             =				tokenizer.encode('''konstruowanie wielu sekwencji'''       ,       add_special_tokens=A_)

         lowerCAmelCase_    :  List[Any]             =				tokenizer.build_inputs_with_special_tokens(A_)
         lowerCAmelCase_    :  str             =				tokenizer.build_inputs_with_special_tokens(A_       ,       A_)

         assert encoded_sentence == [0] + text + [2]
         assert encoded_pair == [0] + text + [2] + text_a + [2]










from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__						:						Dict          =      logging.get_logger(__name__)

A__						:						List[str]          =      {
    '''google/electra-small-generator''': '''https://huggingface.co/google/electra-small-generator/resolve/main/config.json''',
    '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/config.json''',
    '''google/electra-large-generator''': '''https://huggingface.co/google/electra-large-generator/resolve/main/config.json''',
    '''google/electra-small-discriminator''': (
        '''https://huggingface.co/google/electra-small-discriminator/resolve/main/config.json'''
    ),
    '''google/electra-base-discriminator''': (
        '''https://huggingface.co/google/electra-base-discriminator/resolve/main/config.json'''
    ),
    '''google/electra-large-discriminator''': (
        '''https://huggingface.co/google/electra-large-discriminator/resolve/main/config.json'''
    ),
}



class        __snake_case		(  UpperCamelCase_		):
							_a       =				'''electra'''




							def __init__( self  : Dict       ,       A_  : Dict=3_0_5_2_2       ,       A_  : List[Any]=1_2_8       ,       A_  : Dict=2_5_6       ,       A_  : Any=1_2       ,       A_  : Optional[Any]=4       ,       A_  : Optional[int]=1_0_2_4       ,       A_  : Optional[int]="gelu"       ,       A_  : str=0.1       ,       A_  : Optional[int]=0.1       ,       A_  : str=5_1_2       ,       A_  : Optional[int]=2       ,       A_  : Dict=0.02       ,       A_  : Union[str, Any]=1e-12       ,       A_  : List[Any]="first"       ,       A_  : List[str]=True       ,       A_  : List[str]="gelu"       ,       A_  : Any=0.1       ,       A_  : int=0       ,       A_  : Optional[Any]="absolute"       ,       A_  : Optional[int]=True       ,       A_  : str=None       ,       **A_  : List[Any]       ,       ):
									super().__init__(pad_token_id=A_       ,       **A_)

									lowerCAmelCase_    :  str             =				vocab_size
									lowerCAmelCase_    :  Union[str, Any]             =				embedding_size
									lowerCAmelCase_    :  Optional[int]             =				hidden_size
									lowerCAmelCase_    :  List[Any]             =				num_hidden_layers
									lowerCAmelCase_    :  List[str]             =				num_attention_heads
									lowerCAmelCase_    :  str             =				intermediate_size
									lowerCAmelCase_    :  Union[str, Any]             =				hidden_act
									lowerCAmelCase_    :  Union[str, Any]             =				hidden_dropout_prob
									lowerCAmelCase_    :  Tuple             =				attention_probs_dropout_prob
									lowerCAmelCase_    :  Optional[Any]             =				max_position_embeddings
									lowerCAmelCase_    :  Dict             =				type_vocab_size
									lowerCAmelCase_    :  Any             =				initializer_range
									lowerCAmelCase_    :  int             =				layer_norm_eps

									lowerCAmelCase_    :  Union[str, Any]             =				summary_type
									lowerCAmelCase_    :  str             =				summary_use_proj
									lowerCAmelCase_    :  str             =				summary_activation
									lowerCAmelCase_    :  List[Any]             =				summary_last_dropout
									lowerCAmelCase_    :  Optional[Any]             =				position_embedding_type
									lowerCAmelCase_    :  List[Any]             =				use_cache
									lowerCAmelCase_    :  Any             =				classifier_dropout





class        __snake_case		(  UpperCamelCase_		):




							@property
							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									if self.task == "multiple-choice":
											lowerCAmelCase_    :  Any             =				{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
									else:
											lowerCAmelCase_    :  Union[str, Any]             =				{0: '''batch''', 1: '''sequence'''}
									return OrderedDict(
									    [
									        ('''input_ids''', dynamic_axis),
									        ('''attention_mask''', dynamic_axis),
									        ('''token_type_ids''', dynamic_axis),
									    ])





