code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  from collections import OrderedDict
  from typing import Any, Mapping, Optional
  
  from ... import PreTrainedTokenizer
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
  from ...onnx.utils import compute_effective_axis_dimension
  from ...utils import TensorType, is_torch_available, logging
  
  
  _lowerCamelCase							:		Dict										=						logging.get_logger(__name__)
  
  _lowerCamelCase							:		List[str]										=						{
      'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json',
      # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
  }
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				"""m2m_100"""
  					lowercase__					=				["""past_key_values"""]
  					lowercase__					=				{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}
  
  
  
  
  
  
  					def __init__(					self,						__UpperCamelCase=128_112,						__UpperCamelCase=1_024,						__UpperCamelCase=12,						__UpperCamelCase=4_096,						__UpperCamelCase=16,						__UpperCamelCase=12,						__UpperCamelCase=4_096,						__UpperCamelCase=16,						__UpperCamelCase=0.05,						__UpperCamelCase=0.05,						__UpperCamelCase=True,						__UpperCamelCase=True,						__UpperCamelCase="relu",						__UpperCamelCase=1_024,						__UpperCamelCase=0.1,						__UpperCamelCase=0.1,						__UpperCamelCase=0.0,						__UpperCamelCase=0.02,						__UpperCamelCase=2,						__UpperCamelCase=True,						__UpperCamelCase=1,						__UpperCamelCase=0,						__UpperCamelCase=2,						**__UpperCamelCase,						):
  							_snake_case			 = vocab_size
  							_snake_case			 = max_position_embeddings
  							_snake_case			 = d_model
  							_snake_case			 = encoder_ffn_dim
  							_snake_case			 = encoder_layers
  							_snake_case			 = encoder_attention_heads
  							_snake_case			 = decoder_ffn_dim
  							_snake_case			 = decoder_layers
  							_snake_case			 = decoder_attention_heads
  							_snake_case			 = dropout
  							_snake_case			 = attention_dropout
  							_snake_case			 = activation_dropout
  							_snake_case			 = activation_function
  							_snake_case			 = init_std
  							_snake_case			 = encoder_layerdrop
  							_snake_case			 = decoder_layerdrop
  							_snake_case			 = use_cache
  							_snake_case			 = encoder_layers
  							_snake_case			 = scale_embedding  # scale factor will be sqrt(d_model) if True
  
  							super().__init__(
  							    pad_token_id=__UpperCamelCase,						bos_token_id=__UpperCamelCase,						eos_token_id=__UpperCamelCase,						is_encoder_decoder=__UpperCamelCase,						decoder_start_token_id=__UpperCamelCase,						**__UpperCamelCase,						)
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					@property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = OrderedDict(
  							    [
  							        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
  							        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
  							    ]				)
  
  							if self.use_past:
  									_snake_case			 = {0: "batch"}
  									_snake_case			 = {0: "batch", 1: "past_decoder_sequence + sequence"}
  							else:
  									_snake_case			 = {0: "batch", 1: "decoder_sequence"}
  									_snake_case			 = {0: "batch", 1: "decoder_sequence"}
  
  							if self.use_past:
  									self.fill_with_past_key_values_(__UpperCamelCase,						direction="inputs"				)
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							# Copied from OnnxConfig.generate_dummy_inputs
  							# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
  							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
  							_snake_case			 = compute_effective_axis_dimension(
  							    __UpperCamelCase,						fixed_dimension=OnnxConfig.default_fixed_batch,						num_token_to_add=0				)
  
  							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
  							_snake_case			 = tokenizer.num_special_tokens_to_add(__UpperCamelCase				)
  							_snake_case			 = compute_effective_axis_dimension(
  							    __UpperCamelCase,						fixed_dimension=OnnxConfig.default_fixed_sequence,						num_token_to_add=__UpperCamelCase				)
  
  							# Generate dummy inputs according to compute batch and sequence
  							_snake_case			 = [" ".join([tokenizer.unk_token]				) * seq_length] * batch_size
  							_snake_case			 = dict(tokenizer(__UpperCamelCase,						return_tensors=__UpperCamelCase				)				)
  							return common_inputs
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = -1,						__UpperCamelCase = -1,						__UpperCamelCase = False,						__UpperCamelCase = None,						):
  							_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  							    __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				)
  
  							# Generate decoder inputs
  							_snake_case			 = seq_length if not self.use_past else 1
  							_snake_case			 = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
  							    __UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				)
  							_snake_case			 = {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
  							_snake_case			 = dict(**__UpperCamelCase,						**__UpperCamelCase				)
  
  							if self.use_past:
  									if not is_torch_available():
  											raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
  									else:
  											import torch
  									_snake_case ,					_snake_case			 = common_inputs["input_ids"].shape
  									_snake_case			 = common_inputs["decoder_input_ids"].shape[1]
  									_snake_case ,					_snake_case			 = self.num_attention_heads
  									_snake_case			 = (
  									    batch,
  									    num_encoder_attention_heads,
  									    encoder_seq_length,
  									    self._config.hidden_size // num_encoder_attention_heads,
  									)
  									_snake_case			 = decoder_seq_length + 3
  									_snake_case			 = (
  									    batch,
  									    num_decoder_attention_heads,
  									    decoder_past_length,
  									    self._config.hidden_size // num_decoder_attention_heads,
  									)
  
  									_snake_case			 = torch.cat(
  									    [common_inputs["decoder_attention_mask"], torch.ones(__UpperCamelCase,						__UpperCamelCase				)],						dim=1				)
  
  									_snake_case			 = []
  									# If the number of encoder and decoder layers are present in the model configuration, both are considered
  									_snake_case ,					_snake_case			 = self.num_layers
  									_snake_case			 = min(__UpperCamelCase,						__UpperCamelCase				)
  									_snake_case			 = max(__UpperCamelCase,						__UpperCamelCase				) - min_num_layers
  									_snake_case			 = "encoder" if num_encoder_layers > num_decoder_layers else "decoder"
  
  									for _ in range(__UpperCamelCase				):
  											common_inputs["past_key_values"].append(
  											    (
  											        torch.zeros(__UpperCamelCase				),
  											        torch.zeros(__UpperCamelCase				),
  											        torch.zeros(__UpperCamelCase				),
  											        torch.zeros(__UpperCamelCase				),
  											    )				)
  									# TODO: test this.
  									_snake_case			 = encoder_shape if remaining_side_name == "encoder" else decoder_shape
  									for _ in range(__UpperCamelCase,						__UpperCamelCase				):
  											common_inputs["past_key_values"].append((torch.zeros(__UpperCamelCase				), torch.zeros(__UpperCamelCase				))				)
  							return common_inputs
  
  
  
  
  
  
  					lowercase__					=				_generate_dummy_inputs_for_default_and_seqaseq_lm
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class   lowerCamelCase_							( UpperCAmelCase_, unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				GPTNeoXJapaneseTokenizer
  					lowercase__					=				False
  					lowercase__					=				{"""do_clean_text""": False, """add_prefix_space""": False}
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							super().setUp()
  
  							_snake_case			 = [
  							    "こん",
  							    "こんに",
  							    "にちは",
  							    "ばんは",
  							    "世界,㔺界",
  							    "、",
  							    "。",
  							    "<BR>",
  							    "<SP>",
  							    "<TAB>",
  							    "<URL>",
  							    "<EMAIL>",
  							    "<TEL>",
  							    "<DATE>",
  							    "<PRICE>",
  							    "<BLOCK>",
  							    "<KIGOU>",
  							    "<U2000U2BFF>",
  							    "<|emoji1|>",
  							    "<unk>",
  							    "<|startoftext|>",
  							    "<|endoftext|>",
  							]
  							_snake_case			 = {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
  							_snake_case			 = {"unk_token": "<unk>"}
  
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["vocab_file"]				)
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["emoji_file"]				)
  							with open(self.vocab_file,						"w",						encoding="utf-8"				) as vocab_writer:
  									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)
  							with open(self.emoji_file,						"w"				) as emoji_writer:
  									emoji_writer.write(json.dumps(__UpperCamelCase				)				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
  							kwargs.update(self.special_tokens_map				)
  							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname,						**__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、㔺界。😀"
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、世界。😀"
  							return input_text, output_text
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case ,					_snake_case			 = self.get_input_output_texts(__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode(__UpperCamelCase,						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.decode(__UpperCamelCase,						clean_up_tokenization_spaces=__UpperCamelCase				)
  							return text, ids
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.get_tokenizer()
  
  							# Testing tokenization
  							_snake_case			 = "こんにちは、世界。　こんばんは、㔺界。"
  							_snake_case			 = ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
  							_snake_case			 = tokenizer.tokenize(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids without special tokens
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids with special tokens
  							_snake_case			 = tokens + [tokenizer.unk_token]
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  
  
  
  
  
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"				)
  
  							_snake_case			 = tokenizer.encode("ありがとう。",						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode("どういたしまして。",						add_special_tokens=__UpperCamelCase				)
  
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				)
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase,						__UpperCamelCase				)
  
  							assert encoded_sentence == ids_a
  							assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# Intentionally convert some words to accommodate character fluctuations unique to Japanese
  							pass
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# tokenizer has no padding token
  							pass
  
  
  
  
  
  