


'''simple docstring'''




import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
				import sqlitea

				import sqlalchemy


__lowercase  :      Dict      		=				datasets.utils.logging.get_logger(__name__)





@dataclass
class _snake_case   ( datasets.BuilderConfig   ):
							"""simple docstring"""





							__a												=     None
							__a												=     None
							__a												=     None
							__a												=     True
							__a												=     None
							__a												=     None
							__a												=     None
							__a												=     1_0_0_0_0
							__a												=     None







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[str]:



											"""simple docstring"""







											if self.sql is None:
															raise ValueError("sql must be specified"	)
											if self.con is None:
															raise ValueError("con must be specified"	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A			,				__A = None			,				)     ->str:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					config_kwargs.copy()
											# We need to stringify the Selectable object to make its hash deterministic

											# The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
											__lowerCamelCase		:      Union[str, Any]											=					config_kwargs["sql"]
											if not isinstance(__A			,				__A	):
															if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
																			import sqlalchemy

																			if isinstance(__A			,				sqlalchemy.sql.Selectable	):
																							__lowerCamelCase		:      Any											=					sqlalchemy.create_engine(config_kwargs["con"].split("://"	)[0] + "://"	)
																							__lowerCamelCase		:      str											=					str(sql.compile(dialect=engine.dialect	)	)
																							__lowerCamelCase		:      Dict											=					sql_str
																			else:
																							raise TypeError(
																							    f'Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(__A	)}: {sql}'	)
															else:
																			raise TypeError(
																			    f'Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(__A	)}: {sql}'	)
											__lowerCamelCase		:      Dict											=					config_kwargs["con"]
											if not isinstance(__A			,				__A	):
															__lowerCamelCase		:      List[Any]											=					id(__A	)
															logger.info(
															    f'SQL connection \'con\' of type {type(__A	)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'	)

											return super().create_config_id(__A			,				custom_features=__A	)







							@property
							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Tuple											=					{
											    "index_col": self.index_col,
											    "columns": self.columns,
											    "params": self.params,
											    "coerce_float": self.coerce_float,
											    "parse_dates": self.parse_dates,
											}
											return pd_read_sql_kwargs





class _snake_case   ( datasets.ArrowBasedBuilder   ):
							"""simple docstring"""





							__a												=     SqlConfig







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Union[str, Any]:



											"""simple docstring"""







											return datasets.DatasetInfo(features=self.config.features	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->str:



											"""simple docstring"""







											return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,				gen_kwargs={}	)]







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->pa.Table:



											"""simple docstring"""







											if self.config.features is not None:
															__lowerCamelCase		:      int											=					self.config.features.arrow_schema
															if all(not require_storage_cast(__A	) for feature in self.config.features.values()	):
																			# cheaper cast
																			__lowerCamelCase		:      Dict											=					pa.Table.from_arrays([pa_table[field.name] for field in schema]			,				schema=__A	)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			__lowerCamelCase		:      Dict											=					table_cast(__A			,				__A	)
											return pa_table







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Dict:



											"""simple docstring"""







											__lowerCamelCase		:      str											=					self.config.chunksize
											__lowerCamelCase		:      str											=					pd.read_sql(
											    self.config.sql			,				self.config.con			,				chunksize=__A			,				**self.config.pd_read_sql_kwargs	)
											__lowerCamelCase		:      Union[str, Any]											=					[sql_reader] if chunksize is None else sql_reader
											for chunk_idx, df in enumerate(__A	):
															__lowerCamelCase		:      Tuple											=					pa.Table.from_pandas(__A	)
															yield chunk_idx, self._cast_table(__A	)









'''simple docstring'''




import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase  :      Any      		=				get_tests_dir("""fixtures/spiece.model""")





@require_sentencepiece
@require_tokenizers
class _snake_case   ( a__    ,   unittest.TestCase   ):
							"""simple docstring"""





							__a												=     ErnieMTokenizer
							__a												=     False
							__a												=     True
							__a												=     False
							__a												=     False







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Union[str, Any]:



											"""simple docstring"""







											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCamelCase		:      Optional[int]											=					ErnieMTokenizer(__A			,				unk_token="<unk>"			,				pad_token="<pad>"	)
											tokenizer.save_pretrained(self.tmpdirname	)







							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[int]:



											"""simple docstring"""







											__lowerCamelCase		:      Optional[int]											=					"this is a test"
											__lowerCamelCase		:      List[str]											=					"this is a test"
											return input_text, output_text







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:      Any											=					"<pad>"
											__lowerCamelCase		:      Any											=					0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A	)			,				__A	)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A	)			,				__A	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:      Union[str, Any]											=					list(self.get_tokenizer().get_vocab().keys()	)

											self.assertEqual(vocab_keys[0]			,				"<pad>"	)
											self.assertEqual(vocab_keys[1]			,				"<unk>"	)
											self.assertEqual(vocab_keys[-1]			,				"▁eloquent"	)
											self.assertEqual(len(__A	)			,				30000	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Dict:



											"""simple docstring"""







											self.assertEqual(self.get_tokenizer().vocab_size			,				30000	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											if not self.test_rust_tokenizer:
															return

											__lowerCamelCase		:      Union[str, Any]											=					self.get_tokenizer()
											__lowerCamelCase		:      Optional[Any]											=					self.get_rust_tokenizer()

											__lowerCamelCase		:      Dict											=					"I was born in 92000, and this is falsé."

											__lowerCamelCase		:      str											=					tokenizer.tokenize(__A	)
											__lowerCamelCase		:      str											=					rust_tokenizer.tokenize(__A	)
											self.assertListEqual(__A			,				__A	)

											__lowerCamelCase		:      Tuple											=					tokenizer.encode(__A			,				add_special_tokens=__A	)
											__lowerCamelCase		:      Tuple											=					rust_tokenizer.encode(__A			,				add_special_tokens=__A	)
											self.assertListEqual(__A			,				__A	)

											__lowerCamelCase		:      Optional[int]											=					self.get_rust_tokenizer()
											__lowerCamelCase		:      Dict											=					tokenizer.encode(__A	)
											__lowerCamelCase		:      List[str]											=					rust_tokenizer.encode(__A	)
											self.assertListEqual(__A			,				__A	)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Optional[int]:



											"""simple docstring"""







											__lowerCamelCase		:      Tuple											=					ErnieMTokenizer(__A			,				do_lower_case=__A			,				unk_token="<unk>"			,				pad_token="<pad>"	)

											__lowerCamelCase		:      Optional[Any]											=					tokenizer.tokenize("This is a test"	)
											self.assertListEqual(__A			,				["▁this", "▁is", "▁a", "▁test"]	)

											self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)			,				[48, 25, 21, 1289]	)

											__lowerCamelCase		:      Any											=					tokenizer.tokenize("I was born in 92000, and this is falsé."	)
											# ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
											self.assertListEqual(
											    __A			,				["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]	)
											__lowerCamelCase		:      Optional[int]											=					tokenizer.convert_tokens_to_ids(__A	)
											self.assertListEqual(__A			,				[31, 23, 386, 19, 518, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]	)

											__lowerCamelCase		:      Dict											=					tokenizer.convert_ids_to_tokens(__A	)
											self.assertListEqual(
											    __A			,				["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]			,				)







							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:      Union[str, Any]											=					ErnieMTokenizer(__A			,				unk_token="<unk>"			,				pad_token="<pad>"	)

											__lowerCamelCase		:      int											=					tokenizer.encode("sequence builders"	)
											__lowerCamelCase		:      List[Any]											=					tokenizer.encode("multi-sequence build"	)

											__lowerCamelCase		:      Tuple											=					tokenizer.build_inputs_with_special_tokens(__A	)
											__lowerCamelCase		:      Optional[int]											=					tokenizer.build_inputs_with_special_tokens(__A			,				__A	)

											assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
											assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
											    tokenizer.sep_token_id
											] + text_a + [tokenizer.sep_token_id]







							@slow
							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:



											"""simple docstring"""







											__lowerCamelCase		:      Tuple											=					{"input_ids": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 9, 304, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 5, 5, 5, 16, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 6460, 1328, 4589, 42, 122009, 115774, 23, 3559, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__A			,				model_name="susnato/ernie-m-base_pytorch"			,				sequences=[
											        "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
											        "general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
											        "Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained "
											        "models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.",
											        "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
											        "conditioning on both left and right context in all layers.",
											        "The quick brown fox jumps over the lazy dog.",
											    ]			,				)





