


'''simple docstring'''



import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 							__UpperCamelCase				(		lowerCAmelCase_		,      unittest.TestCase					):
	A_				=    RoCBertTokenizer
	A_				=    None
	A_				=    False
	A_				=    True
	A_				=    filter_non_english




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						super().setUp()

						__a			:       List[Any]					    =  ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
						__a			:       Tuple					    =  {}
						__a			:       Tuple					    =  {}
						for i, value in enumerate(__a							):
											__a			:       str					    =  i
											__a			:       int					    =  i
						__a			:       List[Any]					    =  os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['vocab_file']							)
						__a			:       int					    =  os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['word_shape_file']							)
						__a			:       Tuple					    =  os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['word_pronunciation_file']							)
						with open(self.vocab_file						,				'w'						,				encoding='utf-8'							) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)
						with open(self.word_shape_file						,				'w'						,				encoding='utf-8'							) as word_shape_writer:
											json.dump(__a						,				__a						,				ensure_ascii=__a							)
						with open(self.word_pronunciation_file						,				'w'						,				encoding='utf-8'							) as word_pronunciation_writer:
											json.dump(__a						,				__a						,				ensure_ascii=__a							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Any					    =  self.tokenizer_class(self.vocab_file						,				self.word_shape_file						,				self.word_pronunciation_file							)

						__a			:       List[str]					    =  tokenizer.tokenize('你好[SEP]你是谁'							)
						self.assertListEqual(__a						,				['你', '好', '[SEP]', '你', '是', '谁']							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,				[5, 6, 2, 5, 7, 8]							)
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__a							)						,				[5, 6, 2, 5, 7, 8]							)
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__a							)						,				[5, 6, 2, 5, 7, 8]							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)						,				['ah', '\u535A', '\u63A8', 'zz']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Tuple					    =  RoCBertBasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,				['hello', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Tuple					    =  RoCBertBasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hällo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['h\u00E9llo']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Union[str, Any]					    =  RoCBertBasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       str					    =  RoCBertBasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  RoCBertBasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Union[str, Any]					    =  RoCBertBasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       str					    =  RoCBertBasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Union[str, Any]					    =  RoCBertBasicTokenizer(do_lower_case=__a						,				never_split=['[UNK]']							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)						,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       str					    =  ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__a			:       Union[str, Any]					    =  {}
						for i, token in enumerate(__a							):
											__a			:       Any					    =  i
						__a			:       Union[str, Any]					    =  RoCBertWordpieceTokenizer(vocab=__a						,				unk_token='[UNK]'							)

						self.assertListEqual(tokenizer.tokenize(''							)						,				[]							)

						self.assertListEqual(tokenizer.tokenize('unwanted running'							)						,				['un', '##want', '##ed', 'runn', '##ing']							)

						self.assertListEqual(tokenizer.tokenize('unwantedX running'							)						,				['[UNK]', 'runn', '##ing']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_whitespace(' '							)							)
						self.assertTrue(_is_whitespace('\t'							)							)
						self.assertTrue(_is_whitespace('\r'							)							)
						self.assertTrue(_is_whitespace('\n'							)							)
						self.assertTrue(_is_whitespace('\u00A0'							)							)

						self.assertFalse(_is_whitespace('A'							)							)
						self.assertFalse(_is_whitespace('-'							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_control('\u0005'							)							)

						self.assertFalse(_is_control('A'							)							)
						self.assertFalse(_is_control(' '							)							)
						self.assertFalse(_is_control('\t'							)							)
						self.assertFalse(_is_control('\r'							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_punctuation('-'							)							)
						self.assertTrue(_is_punctuation('$'							)							)
						self.assertTrue(_is_punctuation('`'							)							)
						self.assertTrue(_is_punctuation('.'							)							)

						self.assertFalse(_is_punctuation('A'							)							)
						self.assertFalse(_is_punctuation(' '							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__a							) for t in ['Test', '\xad', 'test']]						,				[['[UNK]'], [], ['[UNK]']]							)

						if self.test_rust_tokenizer:
											__a			:       str					    =  self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__a							) for t in ['Test', '\xad', 'test']]						,				[['[UNK]'], [], ['[UNK]']]							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a			:       Union[str, Any]					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       Optional[Any]					    =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																__a			:       Union[str, Any]					    =  tokenizer_r.encode_plus(
																    __a						,				return_attention_mask=__a						,				return_token_type_ids=__a						,				return_offsets_mapping=__a						,				add_special_tokens=__a						,				)

																__a			:       Optional[Any]					    =  tokenizer_r.do_lower_case if hasattr(__a						,				'do_lower_case'							) else False
																__a			:       Any					    =  (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'A'),
																        ((1, 2), ','),
																        ((3, 5), 'na'),
																        ((5, 6), '##ï'),
																        ((6, 8), '##ve'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'Allen'),
																        ((21, 23), '##NL'),
																        ((23, 24), '##P'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'a'),
																        ((1, 2), ','),
																        ((3, 8), 'naive'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'allen'),
																        ((21, 23), '##nl'),
																        ((23, 24), '##p'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]						,				tokenizer_r.convert_ids_to_tokens(tokens['input_ids']							)							)
																self.assertEqual([e[0] for e in expected_results]						,				tokens['offset_mapping']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  ['的', '人', '有']
						__a			:       Optional[int]					    =  ''.join(__a							)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a			:       Optional[int]					    =  True
																__a			:       Tuple					    =  self.tokenizer_class.from_pretrained(__a						,				**__a							)
																__a			:       Optional[Any]					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       Optional[int]					    =  tokenizer_p.encode(__a						,				add_special_tokens=__a							)
																__a			:       Optional[int]					    =  tokenizer_r.encode(__a						,				add_special_tokens=__a							)

																__a			:       int					    =  tokenizer_r.convert_ids_to_tokens(__a							)
																__a			:       Union[str, Any]					    =  tokenizer_p.convert_ids_to_tokens(__a							)

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(__a						,				__a							)
																self.assertListEqual(__a						,				__a							)

																__a			:       List[str]					    =  False
																__a			:       Union[str, Any]					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)
																__a			:       Any					    =  self.tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       Any					    =  tokenizer_r.encode(__a						,				add_special_tokens=__a							)
																__a			:       Optional[int]					    =  tokenizer_p.encode(__a						,				add_special_tokens=__a							)

																__a			:       int					    =  tokenizer_r.convert_ids_to_tokens(__a							)
																__a			:       Union[str, Any]					    =  tokenizer_p.convert_ids_to_tokens(__a							)

																# it is expected that only the first Chinese character is not preceded by "##".
																__a			:       Optional[int]					    =  [
																    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a							)
																]
																self.assertListEqual(__a						,				__a							)
																self.assertListEqual(__a						,				__a							)




	@slow
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[Any]					    =  self.tokenizer_class(self.vocab_file						,				self.word_shape_file						,				self.word_pronunciation_file							)

						__a			:       List[str]					    =  tokenizer.encode('你好'						,				add_special_tokens=__a							)
						__a			:       Optional[int]					    =  tokenizer.encode('你是谁'						,				add_special_tokens=__a							)

						__a			:       Optional[Any]					    =  tokenizer.build_inputs_with_special_tokens(__a							)
						__a			:       Optional[Any]					    =  tokenizer.build_inputs_with_special_tokens(__a						,				__a							)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]





	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[Any]					    =  self.get_tokenizers(do_lower_case=__a							)
						for tokenizer in tokenizers:
											with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																__a			:       Any					    =  '你好，你是谁'
																__a			:       List[Any]					    =  tokenizer.tokenize(__a							)
																__a			:       List[Any]					    =  tokenizer.convert_tokens_to_ids(__a							)
																__a			:       Tuple					    =  tokenizer.convert_tokens_to_shape_ids(__a							)
																__a			:       Optional[Any]					    =  tokenizer.convert_tokens_to_pronunciation_ids(__a							)
																__a			:       List[Any]					    =  tokenizer.prepare_for_model(
																    __a						,				__a						,				__a						,				add_special_tokens=__a							)

																__a			:       str					    =  tokenizer.encode_plus(__a						,				add_special_tokens=__a							)

																self.assertEqual(__a						,				__a							)







import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate







@dataclass(frozen=lowercase__)
class 			_lowercase (							lowercase__):


       """simple docstring"""




       A__											=			field(default="automatic-speech-recognition"							,  metadata={"include_in_asdict_even_if_is_default": True})
       A__											=			Features({"audio": Audio()})
       A__											=			Features({"transcription": Value("string")})
       A__											=			"audio"
       A__											=			"transcription"





       def 						lowerCAmelCase     (   self							:			Any					,      __lowerCamelCase							:			int       ):


             '''simple docstring'''



             if self.audio_column not in features:
                   raise ValueError(f"Column {self.audio_column} is not present in features."       )
             if not isinstance(features[self.audio_column]					,      __lowerCamelCase       ):
                   raise ValueError(f"Column {self.audio_column} is not an Audio type."       )
             lowerCamelCase__		:				Tuple    =  copy.deepcopy(self       )
             lowerCamelCase__		:				Tuple    =  self.input_schema.copy()
             lowerCamelCase__		:				Optional[int]    =  features[self.audio_column]
             lowerCamelCase__		:				int    =  input_schema
             return task_template





       @property
       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             return {self.audio_column: "audio", self.transcription_column: "transcription"}


