




'''simple docstring'''







def 	_lowercase	(			lowercase_		,							lowercase_						):



		'''simple docstring'''







		_snake_case			 =							0
		for ele in arr:
				total_sum += product_sum(lowercase_		,							depth + 1						) if isinstance(lowercase_		,							lowercase_						) else ele
		return total_sum * depth






def 	_lowercase	(			lowercase_						):



		'''simple docstring'''







		return product_sum(lowercase_		,							1						)


if __name__ == "__main__":
						import doctest

						doctest.testmod()











'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class   lowerCamelCase_							( UpperCAmelCase_, unittest.TestCase							):




					'''simple docstring'''






					lowercase__					=				GPTNeoXJapaneseTokenizer
					lowercase__					=				False
					lowercase__					=				{"""do_clean_text""": False, """add_prefix_space""": False}






					def        __SCREAMING_SNAKE_CASE			(					self				):
							super().setUp()

							_snake_case			 = [
							    "こん",
							    "こんに",
							    "にちは",
							    "ばんは",
							    "世界,㔺界",
							    "、",
							    "。",
							    "<BR>",
							    "<SP>",
							    "<TAB>",
							    "<URL>",
							    "<EMAIL>",
							    "<TEL>",
							    "<DATE>",
							    "<PRICE>",
							    "<BLOCK>",
							    "<KIGOU>",
							    "<U2000U2BFF>",
							    "<|emoji1|>",
							    "<unk>",
							    "<|startoftext|>",
							    "<|endoftext|>",
							]
							_snake_case			 = {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
							_snake_case			 = {"unk_token": "<unk>"}

							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["vocab_file"]				)
							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["emoji_file"]				)
							with open(self.vocab_file,						"w",						encoding="utf-8"				) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)
							with open(self.emoji_file,						"w"				) as emoji_writer:
									emoji_writer.write(json.dumps(__UpperCamelCase				)				)






					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
							kwargs.update(self.special_tokens_map				)
							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname,						**__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
							_snake_case			 = "こんにちは、世界。 \nこんばんは、㔺界。😀"
							_snake_case			 = "こんにちは、世界。 \nこんばんは、世界。😀"
							return input_text, output_text






					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
							_snake_case ,					_snake_case			 = self.get_input_output_texts(__UpperCamelCase				)
							_snake_case			 = tokenizer.encode(__UpperCamelCase,						add_special_tokens=__UpperCamelCase				)
							_snake_case			 = tokenizer.decode(__UpperCamelCase,						clean_up_tokenization_spaces=__UpperCamelCase				)
							return text, ids






					def        __SCREAMING_SNAKE_CASE			(					self				):
							pass  # TODO add if relevant






					def        __SCREAMING_SNAKE_CASE			(					self				):
							pass  # TODO add if relevant






					def        __SCREAMING_SNAKE_CASE			(					self				):
							pass  # TODO add if relevant






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = self.get_tokenizer()

							# Testing tokenization
							_snake_case			 = "こんにちは、世界。　こんばんは、㔺界。"
							_snake_case			 = ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
							_snake_case			 = tokenizer.tokenize(__UpperCamelCase				)
							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)

							# Testing conversion to ids without special tokens
							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)

							# Testing conversion to ids with special tokens
							_snake_case			 = tokens + [tokenizer.unk_token]
							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)






					@slow
					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"				)

							_snake_case			 = tokenizer.encode("ありがとう。",						add_special_tokens=__UpperCamelCase				)
							_snake_case			 = tokenizer.encode("どういたしまして。",						add_special_tokens=__UpperCamelCase				)

							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				)
							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase,						__UpperCamelCase				)

							assert encoded_sentence == ids_a
							assert encoded_pair == ids_a + ids_a






					def        __SCREAMING_SNAKE_CASE			(					self				):
							# Intentionally convert some words to accommodate character fluctuations unique to Japanese
							pass






					def        __SCREAMING_SNAKE_CASE			(					self				):
							# tokenizer has no padding token
							pass





