code: 
  
  
  
  
  """simple docstring"""
  
  import unittest
  
  import numpy as np
  
  from transformers import (
      GPTSanJapaneseConfig,
      GPTSanJapaneseForConditionalGeneration,
      GPTSanJapaneseModel,
      GPTSanJapaneseTokenizer,
      is_torch_available,
  )
  from transformers.generation import GenerationConfig
  from transformers.testing_utils import require_torch, slow, tooslow, torch_device
  
  from ...generation.test_utils import GenerationTesterMixin
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  class 			lowerCamelCase__    :
  
  
  			def __init__(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE=36_000				, SCREAMING_SNAKE_CASE=13				, SCREAMING_SNAKE_CASE=7				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE=32				, SCREAMING_SNAKE_CASE=42				, SCREAMING_SNAKE_CASE=5				, SCREAMING_SNAKE_CASE=2				, SCREAMING_SNAKE_CASE=4				, SCREAMING_SNAKE_CASE=2				, SCREAMING_SNAKE_CASE=32				, SCREAMING_SNAKE_CASE=80				, SCREAMING_SNAKE_CASE=33				, SCREAMING_SNAKE_CASE=0.0				, SCREAMING_SNAKE_CASE=1E-6				, SCREAMING_SNAKE_CASE=100				, SCREAMING_SNAKE_CASE=0.0				, ):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[Any]								=				vocab_size
  					snake_case		:						List[Any]								=				parent
  					snake_case		:						int								=				batch_size
  					snake_case		:						Union[str, Any]								=				num_contexts
  					# For common tests
  					snake_case		:						str								=				self.num_contexts
  					snake_case		:						Union[str, Any]								=				is_training
  					snake_case		:						str								=				hidden_size
  					snake_case		:						Tuple								=				num_ext_layers
  					snake_case		:						Union[str, Any]								=				ext_size
  					snake_case		:						List[Any]								=				num_hidden_layers
  					snake_case		:						Union[str, Any]								=				num_attention_heads
  					snake_case		:						Union[str, Any]								=				num_experts
  					snake_case		:						List[str]								=				d_ff
  					snake_case		:						Dict								=				d_ext
  					snake_case		:						List[str]								=				d_spout
  					snake_case		:						List[str]								=				dropout_rate
  					snake_case		:						Optional[Any]								=				layer_norm_epsilon
  					snake_case		:						Optional[Any]								=				expert_capacity
  					snake_case		:						List[str]								=				router_jitter_noise
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					return GPTSanJapaneseConfig.from_pretrained("Tanrei/GPTSAN-japanese"							)
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size							)
  
  					snake_case		:						Optional[Any]								=				self.get_config()
  
  					return (config, input_ids)
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size							)
  
  					snake_case		:						List[str]								=				self.get_config()
  
  					return (config, {"input_ids": input_ids})
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					return GPTSanJapaneseConfig(
  					    vocab_size=self.vocab_size				, num_contexts=self.seq_length				, d_model=self.hidden_size				, d_ff=self.d_ff				, d_ext=self.d_ext				, d_spout=self.d_spout				, num_switch_layers=self.num_hidden_layers - self.num_ext_layers				, num_ext_layers=self.num_ext_layers				, num_heads=self.num_attention_heads				, num_experts=self.num_experts				, expert_capacity=self.expert_capacity				, dropout_rate=self.dropout_rate				, layer_norm_epsilon=self.layer_norm_epsilon				, router_jitter_noise=self.router_jitter_noise				, )
  
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE				, ):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				GPTSanJapaneseForConditionalGeneration(config=SCREAMING_SNAKE_CASE							)
  					model.to(SCREAMING_SNAKE_CASE							)
  					model.eval()
  					snake_case		:						Dict								=				model(
  					    input_ids=SCREAMING_SNAKE_CASE				, )
  					self.parent.assertIsNotNone(SCREAMING_SNAKE_CASE							)
  
  
  
  
  
  
  
  @require_torch
  class 			lowerCamelCase__    (  lowerCamelCase_				,  lowerCamelCase_				,  unittest.TestCase			):
  			a__ :      Tuple								=     (GPTSanJapaneseModel,) if is_torch_available() else ()
  			a__ :      str								=     (
  			    {
  			        """conversational""": GPTSanJapaneseForConditionalGeneration,
  			        """feature-extraction""": GPTSanJapaneseForConditionalGeneration,
  			        """summarization""": GPTSanJapaneseForConditionalGeneration,
  			        """text2text-generation""": GPTSanJapaneseForConditionalGeneration,
  			        """translation""": GPTSanJapaneseForConditionalGeneration,
  			    }
  			    if is_torch_available()
  			    else {}
  			)
  			a__ :      List[Any]								=     False
  			a__ :      Optional[int]								=     False
  			a__ :      Optional[int]								=     False
  			a__ :      int								=     False
  			a__ :      Optional[int]								=     False
  			a__ :      List[Any]								=     False
  			a__ :      str								=     False
  			a__ :      Any								=     False
  			# The small GPTSAN_JAPANESE model needs higher percentages for CPU/MP tests
  			a__ :      Tuple								=     [0.8, 0.9]
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					if pipeline_test_casse_name == "SummarizationPipelineTests":
  							# TODO: fix `_reorder_cache` is not implemented for this model
  							return True
  					elif pipeline_test_casse_name == "Text2TextGenerationPipelineTests":
  							# TODO: check this.
  							return True
  
  					return False
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				GPTSanJapaneseTester(self							)
  					snake_case		:						List[Any]								=				ConfigTester(self				, config_class=SCREAMING_SNAKE_CASE				, d_model=37							)
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					GPTSanJapaneseConfig()
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						List[str]								=				self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE							)
  
  
  			@unittest.skip(
  			    reason="skip for now as the computed `max_memory` by `model_split_percents` in the test method will be changed inside `from_pretrained`"							)
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					super().test_model_parallelism()
  
  
  
  			@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					pass
  
  
  
  
  
  
  
  @require_torch
  class 			lowerCamelCase__    (  lowerCamelCase_				,  lowerCamelCase_				,  unittest.TestCase			):
  			a__ :      Dict								=     (GPTSanJapaneseForConditionalGeneration,) if is_torch_available() else ()
  			a__ :      Optional[Any]								=     False
  			a__ :      Union[str, Any]								=     False
  			a__ :      Optional[int]								=     False
  			a__ :      List[str]								=     False
  			a__ :      List[str]								=     False
  			a__ :      Dict								=     False
  			# The small GPTSAN_JAPANESE model needs higher percentages for CPU/MP tests
  			a__ :      Any								=     [0.8, 0.9]
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				GPTSanJapaneseTester(self							)
  					snake_case		:						Tuple								=				ConfigTester(self				, config_class=SCREAMING_SNAKE_CASE				, d_model=37							)
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					GPTSanJapaneseConfig()
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				self.model_tester.prepare_config_and_inputs()
  					self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE							)
  
  
  			@unittest.skip(
  			    reason="skip for now as the computed `max_memory` by `model_split_percents` in the test method will be changed inside `from_pretrained`"							)
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					super().test_model_parallelism()
  
  
  			@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					pass
  
  
  			@slow
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Dict								=				GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Tuple								=				tokenizer.encode("武田信玄は"				, return_tensors="pt"							)
  					snake_case		:						str								=				model(SCREAMING_SNAKE_CASE							)
  					snake_case		:						str								=				outputs.logits.detach().cpu().numpy()
  					# Output of original model created with mesh-tensoflow
  					snake_case		:						Tuple								=				[
  					    # fmt: off
  					    [-12.0_37_83_98_89_52_63_67, -12.4_33_06_15_99_73_14_45, -14.3_33_84_03_70_17_82_23, -12.4_50_34_59_93_04_19_92, -11.1_66_13_76_95_31_25,
  					    -11.9_30_13_76_34_27_73_44, -10.6_59_74_04_47_99_80_47, -12.9_09_57_45_08_66_69_92, -13.2_41_04_30_90_82_03_12, -13.3_98_57_95_97_47_31_45,
  					    -11.1_07_52_48_71_82_61_72, -12.3_68_59_41_69_61_67, -22.97_94_31_15_23_43_75, -10.4_81_06_76_57_47_07_03, -12.4_84_03_07_23_57_17_77,
  					    -12.8_07_36_06_49_10_88_87, -14.7_69_70_00_50_35_40_04, -12.2_33_57_96_35_62_01_17, -13.4_28_14_54_08_63_03_71, -22.6_24_17_79_32_73_92_58],
  					    [-7.5_11_14_98_83_27_02_64, -8.2_81_85_17_68_49_36_52, -7.9_43_12_71_55_30_39_55, -7.55_02_13_33_69_44_58, -6.49_86_99_66_50_69_58,
  					    -7.5_86_79_62_83_72_19_24, -6.9_78_08_59_94_72_04_59, -7.8_39_14_51_83_56_32_32, -8.21_96_41_68_54_85_84, -8.6_95_09_12_47_55_85_94,
  					    -6.7_06_91_06_10_19_89_75, -6.65_85_79_82_63_54_98_05, -19.5_65_69_86_23_65_72_27, -5.3_53_84_27_35_29_05_27, -8.3_50_68_60_73_30_32_23,
  					    -8.0_39_38_86_56_61_62_11, -10.8_56_56_92_90_16_11_33, -7.75_15_44_47_55_55_42, -8.8_19_02_21_78_64_99_02, -19.51_53_27_45_36_13_28],
  					    [-9.73_06_62_34_58_86_23, -10.2_23_92_27_29_49_21_88, -9.9_32_98_14_91_08_88_67, -11.8_57_83_67_23_32_76_37, -7.6_62_62_62_66_47_94_92,
  					    -11.13_52_91_09_95_48_34, -7.7_65_09_76_18_10_30_27, -11.4_72_92_32_78_80_85_94, -9.5_43_14_99_48_12_01_17, -11.9_05_63_39_26_39_16_02,
  					    -9.3_66_16_42_07_45_84_96, -11.5_73_42_81_53_99_17, -23.6_99_00_32_19_60_44_92, -9.4_29_59_02_25_21_97_27, -10.42_83_92_41_02_78_32,
  					    -10.5_85_24_03_64_07_47_07, -10.94_77_19_57_39_74_61, -11.0_95_41_60_69_03_07_62, -10.3_90_24_06_69_25_04_88, -23.7_69_37_29_40_06_34_77],
  					    [-9.7_28_26_57_62_32_91_02, -9.8_59_71_26_00_70_80_08, -10.09_72_92_90_00_85_45, -9.6_78_52_21_09_98_53_52, -6.8_79_51_99_39_42_26_07,
  					    -9.68_48_75_48_82_81_25, -4.28_03_42_57_88_87_93_95, -10.0_18_91_42_22_71_72_85, -9.3_08_44_59_30_48_09_57, -10.63_39_45_46_50_87_89,
  					    -8.0_83_64_67_74_29_19_92, -9.06_30_14_98_41_30_86, -21.9_04_26_63_57_42_18_75, -8.90_16_08_46_71_02_05, -8.8_41_87_60_29_96_82_62,
  					    -11.8_56_71_99_70_70_31_25, -12.0_79_39_81_55_21_24_02, -11.2_33_75_32_04_34_57_03, -10.1_77_33_86_00_15_86_91, -21.87_25_66_22_31_44_53],
  					    [-9.6_69_76_45_18_73_77_93, -9.6_14_19_86_84_69_23_83, -9.8_14_51_03_45_45_89_84, -9.9_96_50_19_22_60_74_22, -11.3_75_69_04_60_20_50_78,
  					    -10.1_13_40_52_27_66_11_33, -10.5_46_86_73_70_60_54_69, -10.04_36_90_68_14_57_52, -10.9_07_80_92_57_50_73_24, -10.5_04_21_61_94_15_28_32,
  					    -11.1_29_19_90_28_01_51_37, -10.1_51_12_40_00_54_93_16, -21.96_58_67_99_62_15_82, -9.0_86_34_94_87_30_46_88, -11.7_30_33_90_50_29_29_69,
  					    -10.4_60_66_76_10_16_84_57, -10.2_98_04_99_26_75_78_12, -10.7_84_14_82_16_24_75_59, -10.8_40_69_34_73_81_59_18, -22.03_15_28_47_29_00_39],
  					    # fmt: on
  					]
  					snake_case		:						Tuple								=				np.array(SCREAMING_SNAKE_CASE							).flatten()
  					snake_case		:						int								=				output_logits[0, :, :20].flatten()
  
  					def check(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE=5E-4							):
  							return abs(a - b							) < epsilon * max(abs(SCREAMING_SNAKE_CASE							)				, abs(SCREAMING_SNAKE_CASE							)							)
  
  					self.assertTrue(np.all([check(target[i]				, predict[i]							) for i in range(len(SCREAMING_SNAKE_CASE							)							)]							)							)
  
  
  			@slow
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Union[str, Any]								=				GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					model.to(SCREAMING_SNAKE_CASE							)
  
  					# set deterministically
  					snake_case		:						str								=				GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Optional[int]								=				1
  
  					# use different length sentences to test batching
  					snake_case		:						Tuple								=				[
  					    "甲斐なら武田と言うほど",
  					    "織田信長は、",
  					]
  
  					snake_case		:						List[Any]								=				"left"
  					snake_case		:						Dict								=				tokenizer(SCREAMING_SNAKE_CASE				, return_tensors="pt"				, padding=SCREAMING_SNAKE_CASE							)
  					snake_case		:						Tuple								=				inputs["input_ids"].to(SCREAMING_SNAKE_CASE							)
  
  					self.assertNotEqual(inputs["attention_mask"][0].numpy().tolist()				, inputs["attention_mask"][1].numpy().tolist()							)
  
  					snake_case		:						List[Any]								=				model.generate(
  					    input_ids=SCREAMING_SNAKE_CASE				, attention_mask=inputs["attention_mask"].to(SCREAMING_SNAKE_CASE							)				, max_new_tokens=3				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						Any								=				tokenizer(sentences[0]				, return_tensors="pt"							).input_ids.to(SCREAMING_SNAKE_CASE							)
  					snake_case		:						str								=				model.generate(
  					    input_ids=SCREAMING_SNAKE_CASE				, max_new_tokens=3				, generation_config=SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						Union[str, Any]								=				tokenizer(sentences[1]				, return_tensors="pt"							).input_ids.to(SCREAMING_SNAKE_CASE							)
  					snake_case		:						Dict								=				model.generate(input_ids=SCREAMING_SNAKE_CASE				, max_new_tokens=3				, generation_config=SCREAMING_SNAKE_CASE							)
  
  					self.assertNotEqual(inputs_non_padded.shape				, inputs_padded.shape							)
  
  					snake_case		:						Any								=				tokenizer.batch_decode(SCREAMING_SNAKE_CASE				, skip_special_tokens=SCREAMING_SNAKE_CASE							)
  					snake_case		:						List[Any]								=				tokenizer.decode(output_non_padded[0]				, skip_special_tokens=SCREAMING_SNAKE_CASE							)
  					snake_case		:						Optional[Any]								=				tokenizer.decode(output_padded[0]				, skip_special_tokens=SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						List[str]								=				[
  					    "甲斐なら武田と言うほど甲斐の武田",
  					    "織田信長は、このような",
  					]
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, [non_padded_sentence, padded_sentence]							)
  
  
  			@tooslow
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						str								=				GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					# Output of original model created with mesh-tensoflow
  					snake_case		:						int								=				[
  					    ("武田信玄は", 35_675),
  					    ("武田信玄は、", 45),
  					    ("武田信玄は、この", 29),
  					    ("武田信玄は、このよう", 30_642),
  					    ("武田信玄は、このような", 35_680),
  					    ("武田信玄は、このような「", 8_640),
  					    ("武田信玄は、このような「武田", 31_617),
  					    ("武田信玄は、このような「武田家", 30_646),
  					    ("武田信玄は、このような「武田家の", 31_617),
  					    ("武田信玄は、このような「武田家の家", 31_381),
  					]
  					for input, output in target:
  							snake_case		:						Optional[int]								=				tokenizer.encode(SCREAMING_SNAKE_CASE				, return_tensors="pt"							)
  							snake_case		:						Optional[int]								=				model(SCREAMING_SNAKE_CASE							)
  							snake_case		:						Any								=				outputs.logits.detach().cpu().numpy()[0]
  							snake_case		:						str								=				np.argmax(output_logits[-1]							)
  							self.assertEqual(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
  
  
  			@slow
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Union[str, Any]								=				GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						List[str]								=				GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					model.to(SCREAMING_SNAKE_CASE							)
  
  					# set deterministically
  					snake_case		:						Tuple								=				GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Any								=				1
  
  					snake_case		:						Union[str, Any]								=				"武田信玄は、"
  					snake_case		:						List[str]								=				tokenizer(SCREAMING_SNAKE_CASE				, return_tensors="pt"							).input_ids.to(SCREAMING_SNAKE_CASE							)
  					snake_case		:						Optional[int]								=				tokenizer([input_text, input_text]				, return_tensors="pt"							).input_ids.to(SCREAMING_SNAKE_CASE							)
  
  					# spout from uniform and one-hot
  					snake_case		:						Optional[Any]								=				[
  					    # fmt: off
  					    [0.87_88_22_08, 0.38_42_63_96, 0.33_22_02_48, 0.43_89_04_06, 0.16_56_22_52,
  					    0.04_80_39_85, 0.21_15_72  , 0.23_18_84_73, 0.37_15_30_68, 0.7_83_63_77 ,
  					    0.02_16_01_72, 0.38_76_17_19, 0.75_29_07_72, 0.90_19_88_57, 0.34_36_57_77,
  					    0.64_16_81_69, 0.44_31_84_71, 0.14_57_57_46, 0.92_56_28_81, 0.40_81_21_48,
  					    0.29_01_91_22, 0.88_86_15_99, 0.65_52_48_46, 0.43_56_34_56, 0.38_17_71_87,
  					    0.70_83_29_65, 0.81_52_78_92, 0.68_83_28_12, 0.38_83_31_92, 0.4_56_15_22 ,
  					    0.14_82_88_17, 0.47_24_82_13, 0.54_35_73_35, 0.82_00_95_66, 0.1_33_88_84 ,
  					    0.02_75_54_17, 0.19_76_46_77, 0.2_42_20_84 , 0.04_75_76_74, 0.65_40_96_06,
  					    0.0_82_45_89 , 0.03_30_43_83, 0.94_38_76_89, 0.98_76_45_09, 0.82_43_39_01,
  					    0.27_64_67_41, 0.64_90_74_93, 0.76_00_94_06, 0.30_08_79_15, 0.17_90_46_89,
  					    0.41_60_17_14, 0.67_04_63_98, 0.10_42_28_22, 0.08_44_73_74, 0.07_35_43_44,
  					    0.61_42_35_65, 0.70_28_48_66, 0.7_53_23_33 , 0.1_97_20_38 , 0.29_57_56_59,
  					    0.90_58_38_86, 0.29_26_53_07, 0.50_00_01_75, 0.70_40_76_55, 0.88_93_63  ,
  					    0.81_90_44_18, 0.66_82_91_28, 0.64_46_88_15, 0.56_56_37_23, 0.85_60_18_75,
  					    0.94_92_46_72, 0.00_16_67_62, 0.25_22_06_43, 0.74_54_02_19, 0.67_99_32_47,
  					    0.1_54_96_75 , 0.39_38_53_52, 0.92_15_36_07, 0.63_74_59_31, 0.27_75_90_43,
  					    0.84_70_22_95, 0.65_90_42_71, 0.58_67_66_14, 0.8_66_69_36 , 0.39_60_74_38,
  					    0.79_95_49_83, 0.42_22_06_97, 0.39_65_03_81, 0.7_84_98_64 , 0.56_15_02_01,
  					    0.15_67_89_25, 0.14_74_60_32, 0.34_54_21_14, 0.47_02_67_83, 0.11_95_64_89,
  					    0.25_42_14_35, 0.33_78_89_01, 0.68_93_48_42, 0.36_42_46_85, 0.71_73_78_98,
  					    0.38_98_34_49, 0.94_39_37_79, 0.39_57_55_88, 0.36_61_65_53, 0.87_10_46_65,
  					    0.64_63_02_03, 0.22_51_69_05, 0.88_27_08_04, 0.15_03_13_38, 0.75_14_43_45,
  					    0.46_45_90_25, 0.85_39_64_54, 0.86_35_56_43, 0.65_13_98_51, 0.70_26_60_61,
  					    0.30_24_13_89, 0.81_05_64_97, 0.88_86_59_69, 0.38_77_38_07, 0.70_63_58_49,
  					    0.90_71_84_59, 0.43_24_57_89, 0.28_00_06_54, 0.45_93_55_62, 0.08_77_35_19,
  					    0.9_55_21_51 , 0.93_90_15_11, 0.22_48_92_88], # uniform
  					    [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
  					     0., 0., 0., 0., 0., 0., 0., 0.],
  					    # fmt: on
  					]
  
  					snake_case		:						str								=				model.generate(
  					    input_ids=SCREAMING_SNAKE_CASE				, spout=spouts[0]				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						Optional[Any]								=				model.generate(
  					    input_ids=SCREAMING_SNAKE_CASE				, spout=spouts[1]				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						Tuple								=				model.generate(
  					    input_ids=SCREAMING_SNAKE_CASE				, spout=SCREAMING_SNAKE_CASE				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						Dict								=				tokenizer.decode(outputa[0]							)
  					snake_case		:						Any								=				tokenizer.decode(outputa[0]							)
  					snake_case		:						Dict								=				tokenizer.batch_decode(SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						int								=				[
  					    "武田信玄は、武田氏の滅亡後、武田氏の居城であった甲斐武田氏の居城である",
  					    "武田信玄は、武田家の滅亡を防ぐため、武田家の家臣である武田信虎を討",
  					]
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, [outa_sentence, outa_sentence]							)
  
  
  
  			@slow
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						Tuple								=				GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					model.to(SCREAMING_SNAKE_CASE							)
  
  					# set deterministically
  					snake_case		:						List[str]								=				GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese"							)
  					snake_case		:						List[str]								=				1
  
  					snake_case		:						List[str]								=				"武田信玄"
  					snake_case		:						Tuple								=				"織田信長"
  					snake_case		:						List[Any]								=				"は、"
  					snake_case		:						List[str]								=				"が、"
  					snake_case		:						List[str]								=				tokenizer(SCREAMING_SNAKE_CASE				, prefix_text=SCREAMING_SNAKE_CASE				, return_tensors="pt"							)
  					snake_case		:						Optional[int]								=				tokenizer(SCREAMING_SNAKE_CASE				, prefix_text=SCREAMING_SNAKE_CASE				, return_tensors="pt"							)
  					snake_case		:						Union[str, Any]								=				tokenizer([[prefix_text_a, input_text_a], [prefix_text_a, input_text_a]]				, return_tensors="pt"							)
  
  					snake_case		:						int								=				model.generate(
  					    input_ids=input_tok_a.input_ids.to(SCREAMING_SNAKE_CASE							)				, token_type_ids=input_tok_a.token_type_ids.to(SCREAMING_SNAKE_CASE							)				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						str								=				model.generate(
  					    input_ids=input_tok_a.input_ids.to(SCREAMING_SNAKE_CASE							)				, token_type_ids=input_tok_a.token_type_ids.to(SCREAMING_SNAKE_CASE							)				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						List[Any]								=				model.generate(
  					    input_ids=input_tok_a.input_ids.to(SCREAMING_SNAKE_CASE							)				, token_type_ids=input_tok_a.token_type_ids.to(SCREAMING_SNAKE_CASE							)				, attention_mask=input_tok_a.attention_mask.to(SCREAMING_SNAKE_CASE							)				, max_new_tokens=20				, generation_config=SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						str								=				tokenizer.decode(outputa[0]							)
  					snake_case		:						Any								=				tokenizer.decode(outputa[0]							)
  					snake_case		:						Any								=				tokenizer.batch_decode(SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						str								=				[
  					    "武田信玄は、武田氏の祖である武田信虎を、その子・武田信友を擁して",
  					    "織田信長が、織田信長の妻・お市の方を妻として迎えたという逸話が残",
  					]
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
  					self.assertListEqual(SCREAMING_SNAKE_CASE				, [outa_sentence, outa_sentence]							)
  
  
style_context: 
  
  
  
  
  """simple docstring"""
  
  import os
  import re
  import unicodedata
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  
  import sentencepiece as spm
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  __A 		=			logging.get_logger(__name__)
  __A 		=			{"vocab_file": "spiece.model"}
  
  __A 		=			{
      "vocab_file": {
          "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
          "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
      },
  }
  
  __A 		=			{
      "google/fnet-base": 512,
      "google/fnet-large": 512,
  }
  
  __A 		=			"▁"
  class 			lowerCamelCase__    (  lowerCamelCase_			):
  			a__ :      Dict								=     VOCAB_FILES_NAMES
  			a__ :      Tuple								=     PRETRAINED_VOCAB_FILES_MAP
  			a__ :      int								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  			a__ :      List[Any]								=     ["""input_ids""", """token_type_ids"""]
  
  
  			def __init__(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE=False				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE=True				, SCREAMING_SNAKE_CASE="<unk>"				, SCREAMING_SNAKE_CASE="[SEP]"				, SCREAMING_SNAKE_CASE="<pad>"				, SCREAMING_SNAKE_CASE="[CLS]"				, SCREAMING_SNAKE_CASE="[MASK]"				, SCREAMING_SNAKE_CASE = None				, **SCREAMING_SNAKE_CASE				, ):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Dict								=				(
  					    AddedToken(SCREAMING_SNAKE_CASE				, lstrip=SCREAMING_SNAKE_CASE				, rstrip=SCREAMING_SNAKE_CASE				, normalized=SCREAMING_SNAKE_CASE							)
  					    if isinstance(SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE							)
  					    else mask_token
  					)
  
  					snake_case		:						Optional[int]								=				{} if sp_model_kwargs is None else sp_model_kwargs
  
  					super().__init__(
  					    do_lower_case=SCREAMING_SNAKE_CASE				, remove_space=SCREAMING_SNAKE_CASE				, keep_accents=SCREAMING_SNAKE_CASE				, unk_token=SCREAMING_SNAKE_CASE				, sep_token=SCREAMING_SNAKE_CASE				, pad_token=SCREAMING_SNAKE_CASE				, cls_token=SCREAMING_SNAKE_CASE				, mask_token=SCREAMING_SNAKE_CASE				, sp_model_kwargs=self.sp_model_kwargs				, **SCREAMING_SNAKE_CASE				, )
  
  					snake_case		:						List[str]								=				do_lower_case
  					snake_case		:						Any								=				remove_space
  					snake_case		:						Union[str, Any]								=				keep_accents
  					snake_case		:						Optional[int]								=				vocab_file
  
  					snake_case		:						Dict								=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
  					self.sp_model.Load(SCREAMING_SNAKE_CASE							)
  
  
  			@property
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					return len(self.sp_model							)
  
  
  			def 							lowerCamelCase_				(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Optional[int]								=				{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE							): i for i in range(self.vocab_size							)}
  					vocab.update(self.added_tokens_encoder							)
  					return vocab
  
  
  			def __getstate__(     self							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Tuple								=				self.__dict__.copy()
  					snake_case		:						List[Any]								=				None
  					return state
  
  
  			def __setstate__(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Union[str, Any]								=				d
  
  					# for backward compatibility
  					if not hasattr(self				, "sp_model_kwargs"							):
  							snake_case		:						List[Any]								=				{}
  
  					snake_case		:						Tuple								=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
  					self.sp_model.Load(self.vocab_file							)
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					if self.remove_space:
  							snake_case		:						int								=				" ".join(inputs.strip().split()							)
  					else:
  							snake_case		:						Union[str, Any]								=				inputs
  					snake_case		:						Union[str, Any]								=				outputs.replace("``"				, "\""							).replace("''"				, "\""							)
  
  					if not self.keep_accents:
  							snake_case		:						List[Any]								=				unicodedata.normalize("NFKD"				, SCREAMING_SNAKE_CASE							)
  							snake_case		:						int								=				"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE							)]							)
  					if self.do_lower_case:
  							snake_case		:						Union[str, Any]								=				outputs.lower()
  
  					return outputs
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						List[str]								=				self.preprocess_text(SCREAMING_SNAKE_CASE							)
  					snake_case		:						List[str]								=				self.sp_model.encode(SCREAMING_SNAKE_CASE				, out_type=SCREAMING_SNAKE_CASE							)
  					snake_case		:						Optional[Any]								=				[]
  					for piece in pieces:
  							if len(SCREAMING_SNAKE_CASE							) > 1 and piece[-1] == str(","							) and piece[-2].isdigit():
  									snake_case		:						Union[str, Any]								=				self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE				, ""							)							)
  									if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
  											if len(cur_pieces[0]							) == 1:
  													snake_case		:						List[Any]								=				cur_pieces[1:]
  											else:
  													snake_case		:						Dict								=				cur_pieces[0][1:]
  									cur_pieces.append(piece[-1]							)
  									new_pieces.extend(SCREAMING_SNAKE_CASE							)
  							else:
  									new_pieces.append(SCREAMING_SNAKE_CASE							)
  
  					return new_pieces
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE							)
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE							)
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Dict								=				[]
  					snake_case		:						Any								=				""
  					snake_case		:						Dict								=				False
  					for token in tokens:
  							# make sure that special tokens are not decoded using sentencepiece model
  							if token in self.all_special_tokens:
  									if not prev_is_special:
  											out_string += " "
  									out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE							) + token
  									snake_case		:						Optional[int]								=				True
  									snake_case		:						Union[str, Any]								=				[]
  							else:
  									current_sub_tokens.append(SCREAMING_SNAKE_CASE							)
  									snake_case		:						Tuple								=				False
  					out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE							)
  					return out_string.strip()
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = False				, SCREAMING_SNAKE_CASE = None				, SCREAMING_SNAKE_CASE = True				, **SCREAMING_SNAKE_CASE				, ):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						int								=				kwargs.pop("use_source_tokenizer"				, SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						Tuple								=				self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE				, skip_special_tokens=SCREAMING_SNAKE_CASE							)
  
  					# To avoid mixing byte-level and unicode for byte-level BPT
  					# we need to build string separately for added tokens and byte-level tokens
  					# cf. https://github.com/huggingface/transformers/issues/1133
  					snake_case		:						Any								=				[]
  					snake_case		:						Union[str, Any]								=				[]
  					for token in filtered_tokens:
  							if skip_special_tokens and token in self.all_special_ids:
  									continue
  							if token in self.added_tokens_encoder:
  									if current_sub_text:
  											sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE							)							)
  											snake_case		:						Optional[Any]								=				[]
  									sub_texts.append(SCREAMING_SNAKE_CASE							)
  							else:
  									current_sub_text.append(SCREAMING_SNAKE_CASE							)
  					if current_sub_text:
  							sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE							)							)
  
  					# Mimic the behavior of the Rust tokenizer:
  					# No space after <unk>
  					if spaces_between_special_tokens:
  							snake_case		:						Any								=				re.sub(r"(<unk>) "				, r"\1"				, " ".join(SCREAMING_SNAKE_CASE							)							)
  					else:
  							snake_case		:						Optional[int]								=				"".join(SCREAMING_SNAKE_CASE							)
  
  					snake_case		:						Optional[int]								=				(
  					    clean_up_tokenization_spaces
  					    if clean_up_tokenization_spaces is not None
  					    else self.clean_up_tokenization_spaces
  					)
  					if clean_up_tokenization_spaces:
  							snake_case		:						List[Any]								=				self.clean_up_tokenization(SCREAMING_SNAKE_CASE							)
  							return clean_text
  					else:
  							return text
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Tuple								=				[self.sep_token_id]
  					snake_case		:						List[str]								=				[self.cls_token_id]
  					if token_ids_a is None:
  							return cls + token_ids_a + sep
  					return cls + token_ids_a + sep + token_ids_a + sep
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None				, SCREAMING_SNAKE_CASE = False							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					if already_has_special_tokens:
  							return super().get_special_tokens_mask(
  							    token_ids_a=SCREAMING_SNAKE_CASE				, token_ids_a=SCREAMING_SNAKE_CASE				, already_has_special_tokens=SCREAMING_SNAKE_CASE							)
  
  					if token_ids_a is not None:
  							return [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1]
  					return [1] + ([0] * len(SCREAMING_SNAKE_CASE							)) + [1]
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					snake_case		:						Dict								=				[self.sep_token_id]
  					snake_case		:						Optional[int]								=				[self.cls_token_id]
  
  					if token_ids_a is None:
  							return len(cls + token_ids_a + sep							) * [0]
  					return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]
  
  
  
  			def 							lowerCamelCase_				(     self				, SCREAMING_SNAKE_CASE				, SCREAMING_SNAKE_CASE = None							):
  
  
  
  					"""simple docstring"""
  
  
  
  
  
  
  
  					if not os.path.isdir(SCREAMING_SNAKE_CASE							):
  							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
  							return
  					snake_case		:						Tuple								=				os.path.join(
  					    SCREAMING_SNAKE_CASE				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)
  
  					if os.path.abspath(self.vocab_file							) != os.path.abspath(SCREAMING_SNAKE_CASE							) and os.path.isfile(self.vocab_file							):
  							copyfile(self.vocab_file				, SCREAMING_SNAKE_CASE							)
  					elif not os.path.isfile(self.vocab_file							):
  							with open(SCREAMING_SNAKE_CASE				, "wb"							) as fi:
  									snake_case		:						List[str]								=				self.sp_model.serialized_model_proto()
  									fi.write(SCREAMING_SNAKE_CASE							)
  
  					return (out_vocab_file,)
  
  