






'''simple docstring'''







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class _lowercase      (							_lowercase							, unittest.TestCase						):
							a							 =			HerbertTokenizer
							a							 =			HerbertTokenizerFast
							a							 =			True




							def 						lowerCamelCase_							(				self:							int	):
														super().setUp()

														# Use a simpler test file without japanese/chinese characters
														with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''     ,				encoding="""utf-8"""	) as f_data:
																					lowerCamelCase__   :      int			    =			f_data.read().replace("""\n\n"""     ,				"""\n"""	).strip()

														lowerCamelCase__   :      str			    =			[
														    """<s>""",
														    """</s>""",
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """w</w>""",
														    """r</w>""",
														    """t</w>""",
														    """lo""",
														    """low""",
														    """er</w>""",
														    """low</w>""",
														    """lowest</w>""",
														    """newer</w>""",
														    """wider</w>""",
														    """,</w>""",
														    """<unk>""",
														]
														lowerCamelCase__   :      Optional[int]			    =			dict(zip(UpperCamelCase__     ,				range(len(UpperCamelCase__	)	)	)	)
														lowerCamelCase__   :      Tuple			    =			["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

														lowerCamelCase__   :      Union[str, Any]			    =			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES["""vocab_file"""]	)
														lowerCamelCase__   :      Union[str, Any]			    =			os.path.join(self.tmpdirname     ,				VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file     ,				"""w"""	) as fp:
																					fp.write(json.dumps(UpperCamelCase__	)	)
														with open(self.merges_file     ,				"""w"""	) as fp:
																					fp.write("""\n""".join(UpperCamelCase__	)	)




							def 						lowerCamelCase_							(				self:							List[Any]     ,				UpperCamelCase__:							List[Any]	):
														lowerCamelCase__   :      Union[str, Any]			    =			"""lower newer"""
														lowerCamelCase__   :      Optional[Any]			    =			"""lower newer"""
														return input_text, output_text




							def 						lowerCamelCase_							(				self:							Optional[int]	):
														lowerCamelCase__   :      Tuple			    =			self.tokenizer_class(vocab_file=self.vocab_file     ,				merges_file=self.merges_file	)

														lowerCamelCase__   :      List[str]			    =			"""lower"""
														lowerCamelCase__   :      Union[str, Any]			    =			["""low""", """er</w>"""]
														lowerCamelCase__   :      Any			    =			tokenizer.tokenize(UpperCamelCase__	)
														self.assertListEqual(UpperCamelCase__     ,				UpperCamelCase__	)

														lowerCamelCase__   :      Any			    =			tokens + ["""<unk>"""]
														lowerCamelCase__   :      Optional[int]			    =			[16, 17, 23]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__	)     ,				UpperCamelCase__	)




							def 						lowerCamelCase_							(				self:							Dict	):
														if not self.test_rust_tokenizer:
																					return

														lowerCamelCase__   :      Tuple			    =			self.get_tokenizer()
														lowerCamelCase__   :      Dict			    =			self.get_rust_tokenizer()

														lowerCamelCase__   :      Optional[int]			    =			"""lower,newer"""

														lowerCamelCase__   :      Union[str, Any]			    =			tokenizer.tokenize(UpperCamelCase__	)
														lowerCamelCase__   :      List[str]			    =			rust_tokenizer.tokenize(UpperCamelCase__	)
														self.assertListEqual(UpperCamelCase__     ,				UpperCamelCase__	)

														lowerCamelCase__   :      List[str]			    =			tokenizer.encode(UpperCamelCase__     ,				add_special_tokens=UpperCamelCase__	)
														lowerCamelCase__   :      Any			    =			rust_tokenizer.encode(UpperCamelCase__     ,				add_special_tokens=UpperCamelCase__	)
														self.assertListEqual(UpperCamelCase__     ,				UpperCamelCase__	)

														lowerCamelCase__   :      Optional[Any]			    =			self.get_rust_tokenizer()
														lowerCamelCase__   :      Optional[int]			    =			tokenizer.encode(UpperCamelCase__	)
														lowerCamelCase__   :      Dict			    =			rust_tokenizer.encode(UpperCamelCase__	)
														self.assertListEqual(UpperCamelCase__     ,				UpperCamelCase__	)




							@slow
							def 						lowerCamelCase_							(				self:							Optional[int]	):
														lowerCamelCase__   :      Optional[Any]			    =			self.tokenizer_class.from_pretrained("""allegro/herbert-base-cased"""	)

														lowerCamelCase__   :      Union[str, Any]			    =			tokenizer.encode("""konstruowanie sekwencji"""     ,				add_special_tokens=UpperCamelCase__	)
														lowerCamelCase__   :      Any			    =			tokenizer.encode("""konstruowanie wielu sekwencji"""     ,				add_special_tokens=UpperCamelCase__	)

														lowerCamelCase__   :      Tuple			    =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__	)
														lowerCamelCase__   :      List[Any]			    =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,				UpperCamelCase__	)

														assert encoded_sentence == [0] + text + [2]
														assert encoded_pair == [0] + text + [2] + text_a + [2]









'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A      :     Optional[Any]       	={
    '''configuration_realm''': ['''REALM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RealmConfig'''],
    '''tokenization_realm''': ['''RealmTokenizer'''],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     str       	=['''RealmTokenizerFast''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A      :     List[Any]       	=[
			    '''REALM_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''RealmEmbedder''',
			    '''RealmForOpenQA''',
			    '''RealmKnowledgeAugEncoder''',
			    '''RealmPreTrainedModel''',
			    '''RealmReader''',
			    '''RealmScorer''',
			    '''load_tf_weights_in_realm''',
			]
			_A      :     Union[str, Any]       	=['''RealmRetriever''']


if TYPE_CHECKING:
			from .configuration_realm import REALM_PRETRAINED_CONFIG_ARCHIVE_MAP, RealmConfig
			from .tokenization_realm import RealmTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_realm import RealmTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_realm import (
						    REALM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    RealmEmbedder,
						    RealmForOpenQA,
						    RealmKnowledgeAugEncoder,
						    RealmPreTrainedModel,
						    RealmReader,
						    RealmScorer,
						    load_tf_weights_in_realm,
						)
						from .retrieval_realm import RealmRetriever


else:
			import sys

			_A      :     int       	=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

