





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 					__A    (    lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						BertTokenizer
			lowerCAmelCase_							     =						BertTokenizerFast
			lowerCAmelCase_							     =						True
			lowerCAmelCase_							     =						True
			lowerCAmelCase_							     =						filter_non_english





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				super().setUp()

				lowerCamelCase__								=		[
				    '''[UNK]''',
				    '''[CLS]''',
				    '''[SEP]''',
				    '''[PAD]''',
				    '''[MASK]''',
				    '''want''',
				    '''##want''',
				    '''##ed''',
				    '''wa''',
				    '''un''',
				    '''runn''',
				    '''##ing''',
				    ''',''',
				    '''low''',
				    '''lowest''',
				]
				lowerCamelCase__								=		os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['''vocab_file''']							)
				with open(self.vocab_file       ,  '''w'''       ,  encoding='''utf-8'''							) as vocab_writer:
					vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		'''UNwant\u00E9d,running'''
				lowerCamelCase__								=		'''unwanted, running'''
				return input_text, output_text





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.tokenizer_class(self.vocab_file							)

				lowerCamelCase__								=		tokenizer.tokenize('''UNwant\u00E9d,running'''							)
				self.assertListEqual(__lowerCAmelCase       ,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase							)       ,  [9, 6, 7, 1_2, 1_0, 1_1]							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				if not self.test_rust_tokenizer:
					return

				lowerCamelCase__								=		self.get_tokenizer()
				lowerCamelCase__								=		self.get_rust_tokenizer()

				lowerCamelCase__								=		'''UNwant\u00E9d,running'''

				lowerCamelCase__								=		tokenizer.tokenize(__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.tokenize(__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		tokenizer.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		self.get_rust_tokenizer()
				lowerCamelCase__								=		tokenizer.encode(__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.encode(__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

				# With lower casing
				lowerCamelCase__								=		self.get_tokenizer(do_lower_case=__lowerCAmelCase							)
				lowerCamelCase__								=		self.get_rust_tokenizer(do_lower_case=__lowerCAmelCase							)

				lowerCamelCase__								=		'''UNwant\u00E9d,running'''

				lowerCamelCase__								=		tokenizer.tokenize(__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.tokenize(__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		tokenizer.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

				lowerCamelCase__								=		self.get_rust_tokenizer()
				lowerCamelCase__								=		tokenizer.encode(__lowerCAmelCase							)
				lowerCamelCase__								=		rust_tokenizer.encode(__lowerCAmelCase							)
				self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)       ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)       ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)       ,  ['''hello''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase       ,  strip_accents=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)       ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)       ,  ['''h\u00E9llo''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase       ,  strip_accents=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)       ,  ['''hello''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)       ,  ['''hello''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase       ,  strip_accents=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)       ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase       ,  strip_accents=__lowerCAmelCase							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)       ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer(do_lower_case=__lowerCAmelCase       ,  never_split=['''[UNK]''']							)

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		BasicTokenizer()
				lowerCamelCase__								=		'''a\n\'ll !!to?\'d of, can\'t.'''
				lowerCamelCase__								=		['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
				self.assertListEqual(tokenizer.tokenize(__lowerCAmelCase							)       ,  __lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				lowerCamelCase__								=		{}
				for i, token in enumerate(__lowerCAmelCase							):
					lowerCamelCase__								=		i
				lowerCamelCase__								=		WordpieceTokenizer(vocab=__lowerCAmelCase       ,  unk_token='''[UNK]'''							)

				self.assertListEqual(tokenizer.tokenize(''''''							)       ,  []							)

				self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)       ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

				self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)       ,  ['''[UNK]''', '''runn''', '''##ing''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.assertTrue(_is_whitespace(''' '''							)							)
				self.assertTrue(_is_whitespace('''\t'''							)							)
				self.assertTrue(_is_whitespace('''\r'''							)							)
				self.assertTrue(_is_whitespace('''\n'''							)							)
				self.assertTrue(_is_whitespace('''\u00A0'''							)							)

				self.assertFalse(_is_whitespace('''A'''							)							)
				self.assertFalse(_is_whitespace('''-'''							)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.assertTrue(_is_control('''\u0005'''							)							)

				self.assertFalse(_is_control('''A'''							)							)
				self.assertFalse(_is_control(''' '''							)							)
				self.assertFalse(_is_control('''\t'''							)							)
				self.assertFalse(_is_control('''\r'''							)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.assertTrue(_is_punctuation('''-'''							)							)
				self.assertTrue(_is_punctuation('''$'''							)							)
				self.assertTrue(_is_punctuation('''`'''							)							)
				self.assertTrue(_is_punctuation('''.'''							)							)

				self.assertFalse(_is_punctuation('''A'''							)							)
				self.assertFalse(_is_punctuation(''' '''							)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_tokenizer()
				lowerCamelCase__								=		self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(__lowerCAmelCase							) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]							)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(__lowerCAmelCase							) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]							)





			@slow
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.tokenizer_class.from_pretrained('''bert-base-uncased'''							)

				lowerCamelCase__								=		tokenizer.encode('''sequence builders'''       ,  add_special_tokens=__lowerCAmelCase							)
				lowerCamelCase__								=		tokenizer.encode('''multi-sequence build'''       ,  add_special_tokens=__lowerCAmelCase							)

				lowerCamelCase__								=		tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase							)
				lowerCamelCase__								=		tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase       ,  __lowerCAmelCase							)

				assert encoded_sentence == [1_0_1] + text + [1_0_2]
				assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
						lowerCamelCase__								=		self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,  **__lowerCAmelCase							)

						lowerCamelCase__								=		F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
						lowerCamelCase__								=		tokenizer_r.encode_plus(
						    __lowerCAmelCase       ,  return_attention_mask=__lowerCAmelCase       ,  return_token_type_ids=__lowerCAmelCase       ,  return_offsets_mapping=__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase       ,  )

						lowerCamelCase__								=		tokenizer_r.do_lower_case if hasattr(__lowerCAmelCase       ,  '''do_lower_case'''							) else False
						lowerCamelCase__								=		(
						    [
						        ((0, 0), tokenizer_r.cls_token),
						        ((0, 1), '''A'''),
						        ((1, 2), ''','''),
						        ((3, 5), '''na'''),
						        ((5, 6), '''##ï'''),
						        ((6, 8), '''##ve'''),
						        ((9, 1_5), tokenizer_r.mask_token),
						        ((1_6, 2_1), '''Allen'''),
						        ((2_1, 2_3), '''##NL'''),
						        ((2_3, 2_4), '''##P'''),
						        ((2_5, 3_3), '''sentence'''),
						        ((3_3, 3_4), '''.'''),
						        ((0, 0), tokenizer_r.sep_token),
						    ]
						    if not do_lower_case
						    else [
						        ((0, 0), tokenizer_r.cls_token),
						        ((0, 1), '''a'''),
						        ((1, 2), ''','''),
						        ((3, 8), '''naive'''),
						        ((9, 1_5), tokenizer_r.mask_token),
						        ((1_6, 2_1), '''allen'''),
						        ((2_1, 2_3), '''##nl'''),
						        ((2_3, 2_4), '''##p'''),
						        ((2_5, 3_3), '''sentence'''),
						        ((3_3, 3_4), '''.'''),
						        ((0, 0), tokenizer_r.sep_token),
						    ]
						)

						self.assertEqual(
						    [e[1] for e in expected_results]       ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
						self.assertEqual([e[0] for e in expected_results]       ,  tokens['''offset_mapping''']							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		['''的''', '''人''', '''有''']
				lowerCamelCase__								=		''''''.join(__lowerCAmelCase							)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
						lowerCamelCase__								=		True
						lowerCamelCase__								=		self.tokenizer_class.from_pretrained(__lowerCAmelCase       ,  **__lowerCAmelCase							)
						lowerCamelCase__								=		self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,  **__lowerCAmelCase							)

						lowerCamelCase__								=		tokenizer_p.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
						lowerCamelCase__								=		tokenizer_r.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)

						lowerCamelCase__								=		tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase							)
						lowerCamelCase__								=		tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase							)

						# it is expected that each Chinese character is not preceded by "##"
						self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)
						self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)

						lowerCamelCase__								=		False
						lowerCamelCase__								=		self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,  **__lowerCAmelCase							)
						lowerCamelCase__								=		self.tokenizer_class.from_pretrained(__lowerCAmelCase       ,  **__lowerCAmelCase							)

						lowerCamelCase__								=		tokenizer_r.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)
						lowerCamelCase__								=		tokenizer_p.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							)

						lowerCamelCase__								=		tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase							)
						lowerCamelCase__								=		tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase							)

						# it is expected that only the first Chinese character is not preceded by "##".
						lowerCamelCase__								=		[
						    F'##{token}' if idx != 0 else token for idx, token in enumerate(__lowerCAmelCase							)
						]
						self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)
						self.assertListEqual(__lowerCAmelCase       ,  __lowerCAmelCase							)










import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						"""Speech2TextFeatureExtractor"""
			lowerCAmelCase_							     =						"""Speech2TextTokenizer"""





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				super().__init__(__lowerCAmelCase       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.feature_extractor
				lowerCamelCase__								=		False





			def __call__(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				if self._in_target_context_manager:
					return self.current_processor(*__lowerCAmelCase       ,  **__lowerCAmelCase							)

				if "raw_speech" in kwargs:
					warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''							)
					lowerCamelCase__								=		kwargs.pop('''raw_speech'''							)
				else:
					lowerCamelCase__								=		kwargs.pop('''audio'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		kwargs.pop('''sampling_rate'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		kwargs.pop('''text'''       ,  __lowerCAmelCase							)
				if len(__lowerCAmelCase							) > 0:
					lowerCamelCase__								=		args[0]
					lowerCamelCase__								=		args[1:]

				if audio is None and text is None:
					raise ValueError('''You need to specify either an `audio` or `text` input to process.'''							)

				if audio is not None:
					lowerCamelCase__								=		self.feature_extractor(__lowerCAmelCase       ,  *__lowerCAmelCase       ,  sampling_rate=__lowerCAmelCase       ,  **__lowerCAmelCase							)
				if text is not None:
					lowerCamelCase__								=		self.tokenizer(__lowerCAmelCase       ,  **__lowerCAmelCase							)

				if text is None:
					return inputs
				elif audio is None:
					return encodings
				else:
					lowerCamelCase__								=		encodings['''input_ids''']
					return inputs





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.batch_decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			@contextmanager
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				warnings.warn(
				    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
				    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
				    '''your audio inputs, or in a separate call.'''							)
				lowerCamelCase__								=		True
				lowerCamelCase__								=		self.tokenizer
				yield
				lowerCamelCase__								=		self.feature_extractor
				lowerCamelCase__								=		False



