






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a       =    logging.get_logger(__name__)

_a       =    {
    '''mnaylor/mega-base-wikitext''': '''https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json''',
}





class __lowerCamelCase					( snake_case__):





	"""simple docstring"""






	UpperCamelCase__								= "mega"

	def __init__(				self			,       UpperCAmelCase=3_0522			,       UpperCAmelCase=128			,       UpperCAmelCase=4			,       UpperCAmelCase=256			,       UpperCAmelCase=16			,       UpperCAmelCase=True			,       UpperCAmelCase=64			,       UpperCAmelCase=False			,       UpperCAmelCase=-1			,       UpperCAmelCase=None			,       UpperCAmelCase=True			,       UpperCAmelCase="scalenorm"			,       UpperCAmelCase=True			,       UpperCAmelCase="silu"			,       UpperCAmelCase="softmax"			,       UpperCAmelCase=0.1			,       UpperCAmelCase=0.1			,       UpperCAmelCase=0.1			,       UpperCAmelCase=False			,       UpperCAmelCase=True			,       UpperCAmelCase=256			,       UpperCAmelCase=True			,       UpperCAmelCase=0.1			,       UpperCAmelCase=2048			,       UpperCAmelCase=False			,       UpperCAmelCase=2			,       UpperCAmelCase=0.02			,       UpperCAmelCase=0.2			,       UpperCAmelCase=0.02			,       UpperCAmelCase=1.0			,       UpperCAmelCase=1			,       UpperCAmelCase=0			,       UpperCAmelCase=2			,       UpperCAmelCase="rotary"			,       UpperCAmelCase=None			,       UpperCAmelCase=True			,       UpperCAmelCase=True			,       **UpperCAmelCase			,       ):
							"""simple docstring"""







							super().__init__(pad_token_id=UpperCAmelCase			,       bos_token_id=UpperCAmelCase			,       eos_token_id=UpperCAmelCase			,       **UpperCAmelCase       )

							_UpperCAmelCase							   =     vocab_size
							_UpperCAmelCase							   =     hidden_size
							_UpperCAmelCase							   =     num_hidden_layers
							_UpperCAmelCase							   =     activation
							_UpperCAmelCase							   =     attention_activation
							_UpperCAmelCase							   =     intermediate_size
							_UpperCAmelCase							   =     ema_projection_size
							_UpperCAmelCase							   =     bidirectional
							_UpperCAmelCase							   =     shared_representation_size
							_UpperCAmelCase							   =     use_chunking
							_UpperCAmelCase							   =     chunk_size
							_UpperCAmelCase							   =     truncation
							_UpperCAmelCase							   =     normalize_before_mega
							_UpperCAmelCase							   =     normalization_type
							_UpperCAmelCase							   =     norm_affine
							_UpperCAmelCase							   =     dropout_prob
							_UpperCAmelCase							   =     hidden_dropout_prob
							_UpperCAmelCase							   =     attention_probs_dropout_prob
							_UpperCAmelCase							   =     use_feature_dropout
							_UpperCAmelCase							   =     use_normalized_ffn
							_UpperCAmelCase							   =     nffn_hidden_size
							_UpperCAmelCase							   =     normalize_before_ffn
							_UpperCAmelCase							   =     nffn_activation_dropout_prob
							_UpperCAmelCase							   =     max_positions
							_UpperCAmelCase							   =     add_token_type_embeddings
							_UpperCAmelCase							   =     type_vocab_size
							_UpperCAmelCase							   =     initializer_range
							_UpperCAmelCase							   =     ema_delta_alpha_range
							_UpperCAmelCase							   =     ema_beta_range
							_UpperCAmelCase							   =     ema_gamma_omega_range
							_UpperCAmelCase							   =     relative_positional_bias
							_UpperCAmelCase							   =     use_cache
							_UpperCAmelCase							   =     classifier_dropout
							_UpperCAmelCase							   =     add_lm_hidden_dense_layer
							_UpperCAmelCase							   =     1  # not used but required by Hugging Face





class __lowerCamelCase					( snake_case__):





	"""simple docstring"""






	@property
	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							if self.task == "multiple-choice":
													_UpperCAmelCase							   =     {0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
													_UpperCAmelCase							   =     {0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]       )








import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class __lowerCamelCase					( snake_case__	,			unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__								= GPTNeoXJapaneseTokenizer
	UpperCamelCase__								= False
	UpperCamelCase__								= {"do_clean_text": False, "add_prefix_space": False}

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							super().setUp()

							_UpperCAmelCase							   =     [
							    'こん',
							    'こんに',
							    'にちは',
							    'ばんは',
							    '世界,㔺界',
							    '、',
							    '。',
							    '<BR>',
							    '<SP>',
							    '<TAB>',
							    '<URL>',
							    '<EMAIL>',
							    '<TEL>',
							    '<DATE>',
							    '<PRICE>',
							    '<BLOCK>',
							    '<KIGOU>',
							    '<U2000U2BFF>',
							    '<|emoji1|>',
							    '<unk>',
							    '<|startoftext|>',
							    '<|endoftext|>',
							]
							_UpperCAmelCase							   =     {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
							_UpperCAmelCase							   =     {'unk_token': '<unk>'}

							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['vocab_file']       )
							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['emoji_file']       )
							with open(self.vocab_file			,       'w'			,       encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
							with open(self.emoji_file			,       'w'       ) as emoji_writer:
													emoji_writer.write(json.dumps(UpperCAmelCase       )       )

	def 			UpperCamelCase       (				self			,       **UpperCAmelCase       ):
							"""simple docstring"""







							kwargs.update(self.special_tokens_map       )
							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,       **UpperCAmelCase       )

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、㔺界。😀'
							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、世界。😀'
							return input_text, output_text

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase							   =     self.get_input_output_texts(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode(UpperCAmelCase			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.decode(UpperCAmelCase			,       clean_up_tokenization_spaces=UpperCAmelCase       )
							return text, ids

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.get_tokenizer()

							# Testing tokenization
							_UpperCAmelCase							   =     'こんにちは、世界。　こんばんは、㔺界。'
							_UpperCAmelCase							   =     ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
							_UpperCAmelCase							   =     tokenizer.tokenize(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids without special tokens
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids with special tokens
							_UpperCAmelCase							   =     tokens + [tokenizer.unk_token]
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

	@slow
	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.tokenizer_class.from_pretrained('abeja/gpt-neox-japanese-2.7b'       )

							_UpperCAmelCase							   =     tokenizer.encode('ありがとう。'			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode('どういたしまして。'			,       add_special_tokens=UpperCAmelCase       )

							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,       UpperCAmelCase       )

							assert encoded_sentence == ids_a
							assert encoded_pair == ids_a + ids_a

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass
