



'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class _A     (			UpperCAmelCase__		, unittest.TestCase			):
		_SCREAMING_SNAKE_CASE    :			Dict  					=	RoCBertTokenizer
		_SCREAMING_SNAKE_CASE    :			Dict  					=	None
		_SCREAMING_SNAKE_CASE    :			int  					=	False
		_SCREAMING_SNAKE_CASE    :			Dict  					=	True
		_SCREAMING_SNAKE_CASE    :			Optional[Any]  					=	filter_non_english





		def 							__A				(   self )				->   Optional[Any]:

					'''simple docstring'''
					super().setUp()

					__UpperCAmelCase      :					str   =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
					__UpperCAmelCase      :					List[Any]   =   {}
					__UpperCAmelCase      :					str   =   {}
					for i, value in enumerate(lowerCamelCase__ ):
								__UpperCAmelCase      :					Optional[Any]   =   i
								__UpperCAmelCase      :					str   =   i
					__UpperCAmelCase      :					str   =   os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""] )
					__UpperCAmelCase      :					Dict   =   os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""word_shape_file"""] )
					__UpperCAmelCase      :					Union[str, Any]   =   os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""word_pronunciation_file"""] )
					with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8""" ) as vocab_writer:
								vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )
					with open(self.word_shape_file       ,				"""w"""       ,				encoding="""utf-8""" ) as word_shape_writer:
								json.dump(lowerCamelCase__       ,				lowerCamelCase__       ,				ensure_ascii=lowerCamelCase__ )
					with open(self.word_pronunciation_file       ,				"""w"""       ,				encoding="""utf-8""" ) as word_pronunciation_writer:
								json.dump(lowerCamelCase__       ,				lowerCamelCase__       ,				ensure_ascii=lowerCamelCase__ )





		def 							__A				(   self )				->   List[Any]:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[int]   =   self.tokenizer_class(self.vocab_file       ,				self.word_shape_file       ,				self.word_pronunciation_file )

					__UpperCAmelCase      :					Union[str, Any]   =   tokenizer.tokenize("""你好[SEP]你是谁""" )
					self.assertListEqual(lowerCamelCase__       ,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""] )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,				[5, 6, 2, 5, 7, 8] )
					self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowerCamelCase__ )       ,				[5, 6, 2, 5, 7, 8] )
					self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase__ )       ,				[5, 6, 2, 5, 7, 8] )





		def 							__A				(   self )				->   List[Any]:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[int]   =   RoCBertBasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )       ,				["""ah""", """\u535A""", """\u63A8""", """zz"""] )





		def 							__A				(   self )				->   Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase      :					Dict   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )       ,				["""hello""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,				["""hello"""] )





		def 							__A				(   self )				->   Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[Any]   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       ,				strip_accents=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,				["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,				["""h\u00E9llo"""] )





		def 							__A				(   self )				->   str:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[Any]   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       ,				strip_accents=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,				["""hello"""] )





		def 							__A				(   self )				->   List[str]:

					'''simple docstring'''
					__UpperCAmelCase      :					Dict   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,				["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,				["""hello"""] )





		def 							__A				(   self )				->   int:

					'''simple docstring'''
					__UpperCAmelCase      :					List[Any]   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





		def 							__A				(   self )				->   Dict:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[int]   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       ,				strip_accents=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





		def 							__A				(   self )				->   str:

					'''simple docstring'''
					__UpperCAmelCase      :					List[str]   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       ,				strip_accents=lowerCamelCase__ )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





		def 							__A				(   self )				->   List[str]:

					'''simple docstring'''
					__UpperCAmelCase      :					str   =   RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       ,				never_split=["""[UNK]"""] )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )       ,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )





		def 							__A				(   self )				->   int:

					'''simple docstring'''
					__UpperCAmelCase      :					int   =   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

					__UpperCAmelCase      :					Optional[int]   =   {}
					for i, token in enumerate(lowerCamelCase__ ):
								__UpperCAmelCase      :					Optional[int]   =   i
					__UpperCAmelCase      :					int   =   RoCBertWordpieceTokenizer(vocab=lowerCamelCase__       ,				unk_token="""[UNK]""" )

					self.assertListEqual(tokenizer.tokenize("""""" )       ,				[] )

					self.assertListEqual(tokenizer.tokenize("""unwanted running""" )       ,				["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

					self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )       ,				["""[UNK]""", """runn""", """##ing"""] )





		def 							__A				(   self )				->   Optional[int]:

					'''simple docstring'''
					self.assertTrue(_is_whitespace(""" """ ) )
					self.assertTrue(_is_whitespace("""\t""" ) )
					self.assertTrue(_is_whitespace("""\r""" ) )
					self.assertTrue(_is_whitespace("""\n""" ) )
					self.assertTrue(_is_whitespace("""\u00A0""" ) )

					self.assertFalse(_is_whitespace("""A""" ) )
					self.assertFalse(_is_whitespace("""-""" ) )





		def 							__A				(   self )				->   Union[str, Any]:

					'''simple docstring'''
					self.assertTrue(_is_control("""\u0005""" ) )

					self.assertFalse(_is_control("""A""" ) )
					self.assertFalse(_is_control(""" """ ) )
					self.assertFalse(_is_control("""\t""" ) )
					self.assertFalse(_is_control("""\r""" ) )





		def 							__A				(   self )				->   int:

					'''simple docstring'''
					self.assertTrue(_is_punctuation("""-""" ) )
					self.assertTrue(_is_punctuation("""$""" ) )
					self.assertTrue(_is_punctuation("""`""" ) )
					self.assertTrue(_is_punctuation(""".""" ) )

					self.assertFalse(_is_punctuation("""A""" ) )
					self.assertFalse(_is_punctuation(""" """ ) )





		def 							__A				(   self )				->   Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase      :					Optional[Any]   =   self.get_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(lowerCamelCase__ ) for t in ["""Test""", """\xad""", """test"""]]       ,				[["""[UNK]"""], [], ["""[UNK]"""]] )

					if self.test_rust_tokenizer:
								__UpperCAmelCase      :					str   =   self.get_rust_tokenizer()
								self.assertListEqual(
								    [rust_tokenizer.tokenize(lowerCamelCase__ ) for t in ["""Test""", """\xad""", """test"""]]       ,				[["""[UNK]"""], [], ["""[UNK]"""]] )





		def 							__A				(   self )				->   Dict:

					'''simple docstring'''







					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
											__UpperCAmelCase      :					Any   =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__       ,				**lowerCamelCase__ )

											__UpperCAmelCase      :					List[str]   =   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
											__UpperCAmelCase      :					List[Any]   =   tokenizer_r.encode_plus(
											    lowerCamelCase__       ,				return_attention_mask=lowerCamelCase__       ,				return_token_type_ids=lowerCamelCase__       ,				return_offsets_mapping=lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__       ,				)

											__UpperCAmelCase      :					Tuple   =   tokenizer_r.do_lower_case if hasattr(lowerCamelCase__       ,				"""do_lower_case""" ) else False
											__UpperCAmelCase      :					List[Any]   =   (
											    [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "A"),
											        ((1, 2), ","),
											        ((3, 5), "na"),
											        ((5, 6), "##ï"),
											        ((6, 8), "##ve"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "Allen"),
											        ((21, 23), "##NL"),
											        ((23, 24), "##P"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											    if not do_lower_case
											    else [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "a"),
											        ((1, 2), ","),
											        ((3, 8), "naive"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "allen"),
											        ((21, 23), "##nl"),
											        ((23, 24), "##p"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											)

											self.assertEqual(
											    [e[1] for e in expected_results]       ,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
											self.assertEqual([e[0] for e in expected_results]       ,				tokens["""offset_mapping"""] )





		def 							__A				(   self )				->   Union[str, Any]:

					'''simple docstring'''
					__UpperCAmelCase      :					Dict   =   ["的", "人", "有"]
					__UpperCAmelCase      :					Optional[int]   =   "".join(lowerCamelCase__ )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
											__UpperCAmelCase      :					Any   =   True
											__UpperCAmelCase      :					Dict   =   self.tokenizer_class.from_pretrained(lowerCamelCase__       ,				**lowerCamelCase__ )
											__UpperCAmelCase      :					List[Any]   =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__       ,				**lowerCamelCase__ )

											__UpperCAmelCase      :					Dict   =   tokenizer_p.encode(lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )
											__UpperCAmelCase      :					List[str]   =   tokenizer_r.encode(lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )

											__UpperCAmelCase      :					Optional[Any]   =   tokenizer_r.convert_ids_to_tokens(lowerCamelCase__ )
											__UpperCAmelCase      :					Tuple   =   tokenizer_p.convert_ids_to_tokens(lowerCamelCase__ )

											# it is expected that each Chinese character is not preceded by "##"
											self.assertListEqual(lowerCamelCase__       ,				lowerCamelCase__ )
											self.assertListEqual(lowerCamelCase__       ,				lowerCamelCase__ )

											__UpperCAmelCase      :					List[str]   =   False
											__UpperCAmelCase      :					int   =   self.rust_tokenizer_class.from_pretrained(lowerCamelCase__       ,				**lowerCamelCase__ )
											__UpperCAmelCase      :					List[Any]   =   self.tokenizer_class.from_pretrained(lowerCamelCase__       ,				**lowerCamelCase__ )

											__UpperCAmelCase      :					Optional[int]   =   tokenizer_r.encode(lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )
											__UpperCAmelCase      :					Tuple   =   tokenizer_p.encode(lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )

											__UpperCAmelCase      :					Union[str, Any]   =   tokenizer_r.convert_ids_to_tokens(lowerCamelCase__ )
											__UpperCAmelCase      :					Union[str, Any]   =   tokenizer_p.convert_ids_to_tokens(lowerCamelCase__ )

											# it is expected that only the first Chinese character is not preceded by "##".
											__UpperCAmelCase      :					List[Any]   =   [
											    f'##{token}' if idx != 0 else token for idx, token in enumerate(lowerCamelCase__ )
											]
											self.assertListEqual(lowerCamelCase__       ,				lowerCamelCase__ )
											self.assertListEqual(lowerCamelCase__       ,				lowerCamelCase__ )





		@slow
		def 							__A				(   self )				->   int:

					'''simple docstring'''
					__UpperCAmelCase      :					Tuple   =   self.tokenizer_class(self.vocab_file       ,				self.word_shape_file       ,				self.word_pronunciation_file )

					__UpperCAmelCase      :					Any   =   tokenizer.encode("""你好"""       ,				add_special_tokens=lowerCamelCase__ )
					__UpperCAmelCase      :					List[str]   =   tokenizer.encode("""你是谁"""       ,				add_special_tokens=lowerCamelCase__ )

					__UpperCAmelCase      :					Dict   =   tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
					__UpperCAmelCase      :					List[Any]   =   tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,				lowerCamelCase__ )

					assert encoded_sentence == [1] + text + [2]
					assert encoded_pair == [1] + text + [2] + text_a + [2]





		def 							__A				(   self )				->   List[str]:

					'''simple docstring'''
					__UpperCAmelCase      :					str   =   self.get_tokenizers(do_lower_case=lowerCamelCase__ )
					for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}' ):
											__UpperCAmelCase      :					Optional[Any]   =   "你好，你是谁"
											__UpperCAmelCase      :					int   =   tokenizer.tokenize(lowerCamelCase__ )
											__UpperCAmelCase      :					int   =   tokenizer.convert_tokens_to_ids(lowerCamelCase__ )
											__UpperCAmelCase      :					Any   =   tokenizer.convert_tokens_to_shape_ids(lowerCamelCase__ )
											__UpperCAmelCase      :					int   =   tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase__ )
											__UpperCAmelCase      :					int   =   tokenizer.prepare_for_model(
											    lowerCamelCase__       ,				lowerCamelCase__       ,				lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )

											__UpperCAmelCase      :					Optional[Any]   =   tokenizer.encode_plus(lowerCamelCase__       ,				add_special_tokens=lowerCamelCase__ )

											self.assertEqual(lowerCamelCase__       ,				lowerCamelCase__ )








'''simple docstring'''


def        __lowerCAmelCase       (__lowerCAmelCase = 50      ):
		_UpperCAmelCase						:				Tuple	    =					[[0] * 3 for _ in range(length + 1      )]

		for row_length in range(length + 1      ):
				for tile_length in range(2					,       5      ):
						for tile_start in range(row_length - tile_length + 1      ):
								different_colour_ways_number[row_length][tile_length - 2] += (
								    different_colour_ways_number[row_length - tile_start - tile_length][
								        tile_length - 2
								    ]
								    + 1
								)

		return sum(different_colour_ways_number[length]      )


if __name__ == "__main__":
						print(F'''{solution() = }''')



