

"""simple docstring"""


import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


__A						   =			1E-4

if is_torch_available():
			import torch

			from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
			from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder





@require_torch
class     snake_case	:

				def __init__(	self							:				Optional[int]     ,  UpperCamelCase__							:				int     ,  UpperCamelCase__							:				Tuple=1_6     ,  UpperCamelCase__							:				Union[str, Any]=1_3     ,  UpperCamelCase__							:				List[Any]=7     ,  UpperCamelCase__							:				int=1_4     ,  UpperCamelCase__							:				List[Any]=1_0     ,  UpperCamelCase__							:				int=1_9     ,  UpperCamelCase__							:				Union[str, Any]=5     ,  UpperCamelCase__							:				List[str]=4     ,  UpperCamelCase__							:				List[Any]=True     ,  UpperCamelCase__							:				str=1_6     ,  UpperCamelCase__							:				Optional[int]=2     ,  UpperCamelCase__							:				List[str]=4     ,  UpperCamelCase__							:				int=4     ,  UpperCamelCase__							:				int="gelu"     ,  UpperCamelCase__							:				Any=0.1     ,  UpperCamelCase__							:				Optional[Any]=0.1     ,  UpperCamelCase__							:				Dict=[1, 2, 3, 4, 5]     ,  UpperCamelCase__							:				Optional[Any]=2_5     ,  UpperCamelCase__							:				Union[str, Any]=5     ,  )->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			d_model
						__lowerCAmelCase:       List[Any]          =			parent
						__lowerCAmelCase:       Any          =			batch_size
						__lowerCAmelCase:       Dict          =			prediction_length
						__lowerCAmelCase:       Optional[int]          =			context_length
						__lowerCAmelCase:       int          =			cardinality
						__lowerCAmelCase:       List[str]          =			num_time_features
						__lowerCAmelCase:       Union[str, Any]          =			lags_sequence
						__lowerCAmelCase:       Optional[Any]          =			embedding_dimension
						__lowerCAmelCase:       int          =			is_training
						__lowerCAmelCase:       Dict          =			hidden_size
						__lowerCAmelCase:       Any          =			num_hidden_layers
						__lowerCAmelCase:       Union[str, Any]          =			num_attention_heads
						__lowerCAmelCase:       Dict          =			intermediate_size
						__lowerCAmelCase:       Optional[int]          =			hidden_act
						__lowerCAmelCase:       Union[str, Any]          =			hidden_dropout_prob
						__lowerCAmelCase:       Tuple          =			attention_probs_dropout_prob

						__lowerCAmelCase:       str          =			context_length
						__lowerCAmelCase:       Any          =			prediction_length + label_length
						__lowerCAmelCase:       Tuple          =			label_length

						__lowerCAmelCase:       Tuple          =			moving_average
						__lowerCAmelCase:       Tuple          =			autocorrelation_factor

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						return AutoformerConfig(
						    d_model=self.d_model     ,  encoder_layers=self.num_hidden_layers     ,  decoder_layers=self.num_hidden_layers     ,  encoder_attention_heads=self.num_attention_heads     ,  decoder_attention_heads=self.num_attention_heads     ,  encoder_ffn_dim=self.intermediate_size     ,  decoder_ffn_dim=self.intermediate_size     ,  dropout=self.hidden_dropout_prob     ,  attention_dropout=self.attention_probs_dropout_prob     ,  prediction_length=self.prediction_length     ,  context_length=self.context_length     ,  label_length=self.label_length     ,  lags_sequence=self.lags_sequence     ,  num_time_features=self.num_time_features     ,  num_static_categorical_features=1     ,  cardinality=[self.cardinality]     ,  embedding_dimension=[self.embedding_dimension]     ,  moving_average=self.moving_average     ,  )

				def 							lowercase_     (	self							:				Any     ,  UpperCamelCase__							:				Union[str, Any])->   str:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			config.context_length + max(config.lags_sequence)

						__lowerCAmelCase:       Any          =			ids_tensor([self.batch_size, 1]     ,  config.cardinality[0])
						__lowerCAmelCase:       Tuple          =			floats_tensor([self.batch_size, _past_length, config.num_time_features])
						__lowerCAmelCase:       Optional[Any]          =			floats_tensor([self.batch_size, _past_length])
						__lowerCAmelCase:       Tuple          =			floats_tensor([self.batch_size, _past_length]) > 0.5

						# decoder inputs
						__lowerCAmelCase:       Any          =			floats_tensor([self.batch_size, config.prediction_length, config.num_time_features])
						__lowerCAmelCase:       int          =			floats_tensor([self.batch_size, config.prediction_length])

						__lowerCAmelCase:       Tuple          =			{
						    "past_values": past_values,
						    "static_categorical_features": static_categorical_features,
						    "past_time_features": past_time_features,
						    "past_observed_mask": past_observed_mask,
						    "future_time_features": future_time_features,
						    "future_values": future_values,
						}
						return inputs_dict

				def 							lowercase_     (	self							:				List[Any])->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Dict          =			self.get_config()
						__lowerCAmelCase:       str          =			self.prepare_autoformer_inputs_dict(UpperCamelCase__)
						return config, inputs_dict

				def 							lowercase_     (	self							:				List[str])->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase      , __lowerCAmelCase:       List[Any]          =			self.prepare_config_and_inputs()
						return config, inputs_dict




				def 							lowercase_     (	self							:				Any     ,  UpperCamelCase__							:				int     ,  UpperCamelCase__							:				Any)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			AutoformerModel(config=UpperCamelCase__).to(UpperCamelCase__).eval()
						__lowerCAmelCase:       Dict          =			model(**UpperCamelCase__)

						__lowerCAmelCase:       Any          =			outputs.encoder_last_hidden_state
						__lowerCAmelCase:       Optional[Any]          =			outputs.last_hidden_state

						with tempfile.TemporaryDirectory() as tmpdirname:
								__lowerCAmelCase:       Tuple          =			model.get_encoder()
								encoder.save_pretrained(UpperCamelCase__)
								__lowerCAmelCase:       Any          =			AutoformerEncoder.from_pretrained(UpperCamelCase__).to(UpperCamelCase__)

						__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase:       Tuple          =			model.create_network_inputs(**UpperCamelCase__)
						__lowerCAmelCase      , __lowerCAmelCase:       Union[str, Any]          =			model.decomposition_layer(transformer_inputs[:, : config.context_length, ...])

						__lowerCAmelCase:       List[str]          =			torch.cat(
						    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])     ,  dim=-1     ,  )
						__lowerCAmelCase:       Tuple          =			encoder(inputs_embeds=UpperCamelCase__)[0]
						self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3)

						__lowerCAmelCase:       Any          =			(
						    torch.mean(transformer_inputs[:, : config.context_length, ...]     ,  dim=1)
						    .unsqueeze(1)
						    .repeat(1     ,  config.prediction_length     ,  1)
						)
						__lowerCAmelCase:       Union[str, Any]          =			torch.zeros(
						    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]     ,  device=enc_input.device     ,  )

						__lowerCAmelCase:       Optional[Any]          =			torch.cat(
						    (
						        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)     ,  dim=1),
						        feature[:, config.context_length - config.label_length :, ...],
						    )     ,  dim=-1     ,  )
						__lowerCAmelCase:       Tuple          =			torch.cat(
						    (
						        torch.cat((trend_input[:, -config.label_length :, ...], mean)     ,  dim=1),
						        feature[:, config.context_length - config.label_length :, ...],
						    )     ,  dim=-1     ,  )

						with tempfile.TemporaryDirectory() as tmpdirname:
								__lowerCAmelCase:       int          =			model.get_decoder()
								decoder.save_pretrained(UpperCamelCase__)
								__lowerCAmelCase:       List[str]          =			AutoformerDecoder.from_pretrained(UpperCamelCase__).to(UpperCamelCase__)

						__lowerCAmelCase:       Union[str, Any]          =			decoder(
						    trend=UpperCamelCase__     ,  inputs_embeds=UpperCamelCase__     ,  encoder_hidden_states=UpperCamelCase__     ,  )[0]

						self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3)





@require_torch
class     snake_case	(				__snake_case,							__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      Tuple         =  (AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
				SCREAMING_SNAKE_CASE_   :      int         =  (AutoformerForPrediction,) if is_torch_available() else ()
				SCREAMING_SNAKE_CASE_   :      Optional[Any]         =  {"""feature-extraction""": AutoformerModel} if is_torch_available() else {}
				SCREAMING_SNAKE_CASE_   :      int         =  False
				SCREAMING_SNAKE_CASE_   :      Optional[Any]         =  False
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  False
				SCREAMING_SNAKE_CASE_   :      List[str]         =  False

				def 							lowercase_     (	self							:				Any)->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			AutoformerModelTester(self)
						__lowerCAmelCase:       int          =			ConfigTester(self     ,  config_class=UpperCamelCase__     ,  has_text_modality=UpperCamelCase__)

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						self.config_tester.run_common_tests()

				def 							lowercase_     (	self							:				List[Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase      , __lowerCAmelCase:       int          =			self.model_tester.prepare_config_and_inputs()
						for model_class in self.all_model_classes:
								__lowerCAmelCase:       Any          =			model_class(UpperCamelCase__)

								with tempfile.TemporaryDirectory() as tmpdirname:
										model.save_pretrained(UpperCamelCase__)
										__lowerCAmelCase      , __lowerCAmelCase:       int          =			model_class.from_pretrained(UpperCamelCase__     ,  output_loading_info=UpperCamelCase__)
								self.assertEqual(info["missing_keys"]     ,  [])

				def 							lowercase_     (	self							:				int)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.model_tester.prepare_config_and_inputs_for_common()
						self.model_tester.check_encoder_decoder_model_standalone(*UpperCamelCase__)

				@unittest.skip(reason="Model has no tokens embeddings")
				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						pass

				def 							lowercase_     (	self							:				List[str])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       List[Any]          =			inspect.signature(getattr(UpperCamelCase__     ,  "forward"))
						# The main input is the name of the argument after `self`
						__lowerCAmelCase:       List[Any]          =			list(model_signature.parameters.keys())[1]
						self.assertEqual(AutoformerModel.main_input_name     ,  UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase      , __lowerCAmelCase:       Optional[Any]          =			self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
								__lowerCAmelCase:       List[str]          =			model_class(UpperCamelCase__)
								__lowerCAmelCase:       Any          =			inspect.signature(model.forward)
								# signature.parameters is an OrderedDict => so arg_names order is deterministic
								__lowerCAmelCase:       str          =			[*signature.parameters.keys()]

								__lowerCAmelCase:       Tuple          =			[
								    "past_values",
								    "past_time_features",
								    "past_observed_mask",
								    "static_categorical_features",
								    "static_real_features",
								    "future_values",
								    "future_time_features",
								]

								if model.__class__.__name__ in ["AutoformerForPrediction"]:
										expected_arg_names.append("future_observed_mask")

								expected_arg_names.extend(
								    [
								        "decoder_attention_mask",
								        "head_mask",
								        "decoder_head_mask",
								        "cross_attn_head_mask",
								        "encoder_outputs",
								        "past_key_values",
								        "output_hidden_states",
								        "output_attentions",
								        "use_cache",
								        "return_dict",
								    ])

								self.assertListEqual(arg_names[: len(UpperCamelCase__)]     ,  UpperCamelCase__)

				def 							lowercase_     (	self							:				str)->   Any:



						'''simple docstring'''


						__lowerCAmelCase      , __lowerCAmelCase:       Dict          =			self.model_tester.prepare_config_and_inputs_for_common()
						__lowerCAmelCase:       Optional[Any]          =			True

						__lowerCAmelCase:       Dict          =			getattr(self.model_tester     ,  "seq_length"     ,  UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			getattr(self.model_tester     ,  "decoder_seq_length"     ,  UpperCamelCase__)
						__lowerCAmelCase:       Tuple          =			getattr(self.model_tester     ,  "encoder_seq_length"     ,  UpperCamelCase__)
						__lowerCAmelCase:       Optional[Any]          =			getattr(self.model_tester     ,  "d_model"     ,  UpperCamelCase__)
						__lowerCAmelCase:       List[str]          =			getattr(self.model_tester     ,  "num_attention_heads"     ,  UpperCamelCase__)
						__lowerCAmelCase:       Optional[int]          =			d_model // num_attention_heads

						for model_class in self.all_model_classes:
								__lowerCAmelCase:       str          =			True
								__lowerCAmelCase:       List[str]          =			False
								__lowerCAmelCase:       Any          =			True
								__lowerCAmelCase:       List[Any]          =			model_class(UpperCamelCase__)
								model.to(UpperCamelCase__)
								model.eval()
								with torch.no_grad():
										__lowerCAmelCase:       Dict          =			model(**self._prepare_for_class(UpperCamelCase__     ,  UpperCamelCase__))
								__lowerCAmelCase:       Any          =			outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
								self.assertEqual(len(UpperCamelCase__)     ,  self.model_tester.num_hidden_layers)

								# check that output_attentions also work using config
								del inputs_dict["output_attentions"]
								__lowerCAmelCase:       Union[str, Any]          =			True
								__lowerCAmelCase:       List[str]          =			model_class(UpperCamelCase__)
								model.to(UpperCamelCase__)
								model.eval()
								with torch.no_grad():
										__lowerCAmelCase:       Optional[Any]          =			model(**self._prepare_for_class(UpperCamelCase__     ,  UpperCamelCase__))
								__lowerCAmelCase:       str          =			outputs.encoder_attentions
								self.assertEqual(len(UpperCamelCase__)     ,  self.model_tester.num_hidden_layers)

								self.assertListEqual(
								    list(attentions[0].shape[-3:])     ,  [self.model_tester.num_attention_heads, encoder_seq_length, dim]     ,  )
								__lowerCAmelCase:       Tuple          =			len(UpperCamelCase__)

								__lowerCAmelCase:       Tuple          =			7

								if "last_hidden_state" in outputs:
										correct_outlen += 1

								if "trend" in outputs:
										correct_outlen += 1

								if "past_key_values" in outputs:
										correct_outlen += 1  # past_key_values have been returned

								if "loss" in outputs:
										correct_outlen += 1

								if "params" in outputs:
										correct_outlen += 1

								self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)

								# decoder attentions
								__lowerCAmelCase:       Optional[int]          =			outputs.decoder_attentions
								self.assertIsInstance(UpperCamelCase__     ,  (list, tuple))
								self.assertEqual(len(UpperCamelCase__)     ,  self.model_tester.num_hidden_layers)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:])     ,  [self.model_tester.num_attention_heads, decoder_seq_length, dim]     ,  )

								# cross attentions
								__lowerCAmelCase:       Any          =			outputs.cross_attentions
								self.assertIsInstance(UpperCamelCase__     ,  (list, tuple))
								self.assertEqual(len(UpperCamelCase__)     ,  self.model_tester.num_hidden_layers)
								self.assertListEqual(
								    list(cross_attentions[0].shape[-3:])     ,  [self.model_tester.num_attention_heads, decoder_seq_length, dim]     ,  )

						# Check attention is always last and order is fine
						__lowerCAmelCase:       Tuple          =			True
						__lowerCAmelCase:       Optional[int]          =			True
						__lowerCAmelCase:       str          =			model_class(UpperCamelCase__)
						model.to(UpperCamelCase__)
						model.eval()
						with torch.no_grad():
								__lowerCAmelCase:       str          =			model(**self._prepare_for_class(UpperCamelCase__     ,  UpperCamelCase__))

						self.assertEqual(out_len + 2     ,  len(UpperCamelCase__))

						__lowerCAmelCase:       Union[str, Any]          =			outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

						self.assertEqual(len(UpperCamelCase__)     ,  self.model_tester.num_hidden_layers)
						self.assertListEqual(
						    list(self_attentions[0].shape[-3:])     ,  [self.model_tester.num_attention_heads, encoder_seq_length, dim]     ,  )




				@is_flaky()
				def 							lowercase_     (	self							:				Optional[Any])->   int:



						'''simple docstring'''


						super().test_retain_grad_hidden_states_attentions()






def     a__    (    __SCREAMING_SNAKE_CASE="train-batch.pt" )	->	int:
		__lowerCAmelCase:       Optional[Any]          =			hf_hub_download(repo_id="hf-internal-testing/tourism-monthly-batch"					, filename=__SCREAMING_SNAKE_CASE					, repo_type="dataset" )
		__lowerCAmelCase:       Any          =			torch.load(__SCREAMING_SNAKE_CASE					, map_location=__SCREAMING_SNAKE_CASE )
		return batch





@require_torch
@slow
class     snake_case	(				unittest.TestCase  ):

				def 							lowercase_     (	self							:				List[str])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			AutoformerModel.from_pretrained("huggingface/autoformer-tourism-monthly").to(UpperCamelCase__)
						__lowerCAmelCase:       Tuple          =			prepare_batch()

						with torch.no_grad():
								__lowerCAmelCase:       Union[str, Any]          =			model(
								    past_values=batch["past_values"]     ,  past_time_features=batch["past_time_features"]     ,  past_observed_mask=batch["past_observed_mask"]     ,  static_categorical_features=batch["static_categorical_features"]     ,  future_values=batch["future_values"]     ,  future_time_features=batch["future_time_features"]     ,  )[0]

						__lowerCAmelCase:       Optional[Any]          =			torch.Size(
						    (6_4, model.config.prediction_length + model.config.label_length, model.config.feature_size))
						self.assertEqual(output.shape     ,  UpperCamelCase__)

						__lowerCAmelCase:       int          =			torch.tensor(
						    [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]     ,  device=UpperCamelCase__)
						self.assertTrue(torch.allclose(output[0, :3, :3]     ,  UpperCamelCase__     ,  atol=UpperCamelCase__))

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly").to(UpperCamelCase__)
						__lowerCAmelCase:       Any          =			prepare_batch("val-batch.pt")
						with torch.no_grad():
								__lowerCAmelCase:       List[str]          =			model(
								    past_values=batch["past_values"]     ,  past_time_features=batch["past_time_features"]     ,  past_observed_mask=batch["past_observed_mask"]     ,  static_categorical_features=batch["static_categorical_features"]     ,  ).encoder_last_hidden_state
						__lowerCAmelCase:       Any          =			torch.Size((6_4, model.config.context_length, model.config.d_model))
						self.assertEqual(output.shape     ,  UpperCamelCase__)

						__lowerCAmelCase:       Optional[int]          =			torch.tensor(
						    [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]     ,  device=UpperCamelCase__)
						self.assertTrue(torch.allclose(output[0, :3, :3]     ,  UpperCamelCase__     ,  atol=UpperCamelCase__))




				def 							lowercase_     (	self							:				Optional[Any])->   Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly").to(UpperCamelCase__)
						__lowerCAmelCase:       int          =			prepare_batch("val-batch.pt")
						with torch.no_grad():
								__lowerCAmelCase:       Dict          =			model.generate(
								    static_categorical_features=batch["static_categorical_features"]     ,  past_time_features=batch["past_time_features"]     ,  past_values=batch["past_values"]     ,  future_time_features=batch["future_time_features"]     ,  past_observed_mask=batch["past_observed_mask"]     ,  )
						__lowerCAmelCase:       Tuple          =			torch.Size((6_4, model.config.num_parallel_samples, model.config.prediction_length))
						self.assertEqual(outputs.sequences.shape     ,  UpperCamelCase__)

						__lowerCAmelCase:       Dict          =			torch.tensor([3130.6763, 4056.5293, 7053.0786]     ,  device=UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			outputs.sequences.mean(dim=1)
						self.assertTrue(torch.allclose(mean_prediction[0, -3:]     ,  UpperCamelCase__     ,  rtol=1e-1))





"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[Any]         =  RoCBertTokenizer
				SCREAMING_SNAKE_CASE_   :      Any         =  None
				SCREAMING_SNAKE_CASE_   :      Any         =  False
				SCREAMING_SNAKE_CASE_   :      Dict         =  True
				SCREAMING_SNAKE_CASE_   :      Optional[int]         =  filter_non_english

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						super().setUp()

						__lowerCAmelCase:       Optional[Any]          =			["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
						__lowerCAmelCase:       List[Any]          =			{}
						__lowerCAmelCase:       Dict          =			{}
						for i, value in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       List[Any]          =			i
								__lowerCAmelCase:       Union[str, Any]          =			i
						__lowerCAmelCase:       List[str]          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["vocab_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_shape_file"])
						__lowerCAmelCase:       int          =			os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES["word_pronunciation_file"])
						with open(self.vocab_file     ,  "w"     ,  encoding="utf-8") as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
						with open(self.word_shape_file     ,  "w"     ,  encoding="utf-8") as word_shape_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)
						with open(self.word_pronunciation_file     ,  "w"     ,  encoding="utf-8") as word_pronunciation_writer:
								json.dump(UpperCamelCase__     ,  UpperCamelCase__     ,  ensure_ascii=UpperCamelCase__)

				def 							lowercase_     (	self							:				Any)->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.tokenize("你好[SEP]你是谁")
						self.assertListEqual(UpperCamelCase__     ,  ["你", "好", "[SEP]", "你", "是", "谁"])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)     ,  [5, 6, 2, 5, 7, 8])

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")     ,  ["ah", "\u535A", "\u63A8", "zz"])

				def 							lowercase_     (	self							:				str)->   Dict:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["hello", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Optional[int])->   List[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hällo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["h\u00E9llo"])

				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Tuple          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				str)->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["hallo", "!", "how", "are", "you", "?"])
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo")     ,  ["hello"])

				def 							lowercase_     (	self							:				Any)->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       Union[str, Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[int])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[int]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HäLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Optional[Any])->   Tuple:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  strip_accents=UpperCamelCase__)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")     ,  ["HaLLo", "!", "how", "Are", "yoU", "?"])

				def 							lowercase_     (	self							:				Tuple)->   str:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__     ,  never_split=["[UNK]"])

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")     ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

				def 							lowercase_     (	self							:				List[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       List[str]          =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						__lowerCAmelCase:       int          =			{}
						for i, token in enumerate(UpperCamelCase__):
								__lowerCAmelCase:       Optional[Any]          =			i
						__lowerCAmelCase:       str          =			RoCBertWordpieceTokenizer(vocab=UpperCamelCase__     ,  unk_token="[UNK]")

						self.assertListEqual(tokenizer.tokenize("")     ,  [])

						self.assertListEqual(tokenizer.tokenize("unwanted running")     ,  ["un", "##want", "##ed", "runn", "##ing"])

						self.assertListEqual(tokenizer.tokenize("unwantedX running")     ,  ["[UNK]", "runn", "##ing"])

				def 							lowercase_     (	self							:				Optional[Any])->   Dict:



						'''simple docstring'''


						self.assertTrue(_is_whitespace(" "))
						self.assertTrue(_is_whitespace("\t"))
						self.assertTrue(_is_whitespace("\r"))
						self.assertTrue(_is_whitespace("\n"))
						self.assertTrue(_is_whitespace("\u00A0"))

						self.assertFalse(_is_whitespace("A"))
						self.assertFalse(_is_whitespace("-"))

				def 							lowercase_     (	self							:				Dict)->   Optional[int]:



						'''simple docstring'''


						self.assertTrue(_is_control("\u0005"))

						self.assertFalse(_is_control("A"))
						self.assertFalse(_is_control(" "))
						self.assertFalse(_is_control("\t"))
						self.assertFalse(_is_control("\r"))

				def 							lowercase_     (	self							:				Union[str, Any])->   str:



						'''simple docstring'''


						self.assertTrue(_is_punctuation("-"))
						self.assertTrue(_is_punctuation("$"))
						self.assertTrue(_is_punctuation("`"))
						self.assertTrue(_is_punctuation("."))

						self.assertFalse(_is_punctuation("A"))
						self.assertFalse(_is_punctuation(" "))

				def 							lowercase_     (	self							:				Dict)->   int:



						'''simple docstring'''


						__lowerCAmelCase:       Any          =			self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

						if self.test_rust_tokenizer:
								__lowerCAmelCase:       Any          =			self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(UpperCamelCase__) for t in ["Test", "\xad", "test"]]     ,  [["[UNK]"], [], ["[UNK]"]])

				def 							lowercase_     (	self							:				Dict)->   Any:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       str          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
										__lowerCAmelCase:       Tuple          =			tokenizer_r.encode_plus(
										    UpperCamelCase__     ,  return_attention_mask=UpperCamelCase__     ,  return_token_type_ids=UpperCamelCase__     ,  return_offsets_mapping=UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  )

										__lowerCAmelCase:       str          =			tokenizer_r.do_lower_case if hasattr(UpperCamelCase__     ,  "do_lower_case") else False
										__lowerCAmelCase:       List[Any]          =			(
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "Allen"),
										        ((2_1, 2_3), "##NL"),
										        ((2_3, 2_4), "##P"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 1_5), tokenizer_r.mask_token),
										        ((1_6, 2_1), "allen"),
										        ((2_1, 2_3), "##nl"),
										        ((2_3, 2_4), "##p"),
										        ((2_5, 3_3), "sentence"),
										        ((3_3, 3_4), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]     ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results]     ,  tokens["offset_mapping"])

				def 							lowercase_     (	self							:				Union[str, Any])->   Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:       Optional[Any]          =			["的", "人", "有"]
						__lowerCAmelCase:       int          =			"".join(UpperCamelCase__)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Tuple          =			True
										__lowerCAmelCase:       str          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       Union[str, Any]          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       List[Any]          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Any          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       List[str]          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

										__lowerCAmelCase:       int          =			False
										__lowerCAmelCase:       Any          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			self.tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)
										__lowerCAmelCase:       str          =			tokenizer_p.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       str          =			tokenizer_r.convert_ids_to_tokens(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer_p.convert_ids_to_tokens(UpperCamelCase__)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase:       Dict          =			[
										    f"##{token}" if idx != 0 else token for idx, token in enumerate(UpperCamelCase__)
										]
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)
										self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				@slow
				def 							lowercase_     (	self							:				Optional[Any])->   Any:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.tokenizer_class(self.vocab_file     ,  self.word_shape_file     ,  self.word_pronunciation_file)

						__lowerCAmelCase:       Dict          =			tokenizer.encode("你好"     ,  add_special_tokens=UpperCamelCase__)
						__lowerCAmelCase:       Union[str, Any]          =			tokenizer.encode("你是谁"     ,  add_special_tokens=UpperCamelCase__)

						__lowerCAmelCase:       Tuple          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__)
						__lowerCAmelCase:       List[Any]          =			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]




				def 							lowercase_     (	self							:				Tuple)->   Optional[Any]:



						'''simple docstring'''


						__lowerCAmelCase:       int          =			self.get_tokenizers(do_lower_case=UpperCamelCase__)
						for tokenizer in tokenizers:
								with self.subTest(f"{tokenizer.__class__.__name__}"):
										__lowerCAmelCase:       str          =			"你好，你是谁"
										__lowerCAmelCase:       Dict          =			tokenizer.tokenize(UpperCamelCase__)
										__lowerCAmelCase:       Union[str, Any]          =			tokenizer.convert_tokens_to_ids(UpperCamelCase__)
										__lowerCAmelCase:       Optional[Any]          =			tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__)
										__lowerCAmelCase:       Tuple          =			tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__)
										__lowerCAmelCase:       Dict          =			tokenizer.prepare_for_model(
										    UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										__lowerCAmelCase:       Optional[Any]          =			tokenizer.encode_plus(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__)

										self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__)


