'''simple docstring'''


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 	__SCREAMING_SNAKE_CASE		(			lowerCamelCase_		,      unittest.TestCase			):







  '''simple docstring'''






  lowerCamelCase_						:List[str]		       =						RealmTokenizer
  lowerCamelCase_						:Union[str, Any]		       =						RealmTokenizerFast
  lowerCamelCase_						:Dict		       =						True
  lowerCamelCase_						:Optional[Any]		       =						True
  lowerCamelCase_						:List[str]		       =						filter_non_english


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       super().setUp()

       UpperCAmelCase_				:  Optional[Any]  = [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       UpperCAmelCase_				:  Tuple  = os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['vocab_file']		)
       with open(self.vocab_file ,						'w' ,						encoding='utf-8'		) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = 'UNwant\u00E9d,running'
       UpperCAmelCase_				:  str  = 'unwanted, running'
       return input_text, output_text


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = self.tokenizer_class(self.vocab_file		)

       UpperCAmelCase_				:  List[Any]  = tokenizer.tokenize('UNwant\u00E9d,running'		)
       self.assertListEqual(snake_case_ ,						['un', '##want', '##ed', ',', 'runn', '##ing']		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_		) ,						[9, 6, 7, 1_2, 1_0, 1_1]		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       if not self.test_rust_tokenizer:
            return

       UpperCAmelCase_				:  str  = self.get_tokenizer()
       UpperCAmelCase_				:  Optional[int]  = self.get_rust_tokenizer()

       UpperCAmelCase_				:  Tuple  = 'UNwant\u00E9d,running'

       UpperCAmelCase_				:  int  = tokenizer.tokenize(snake_case_		)
       UpperCAmelCase_				:  Dict  = rust_tokenizer.tokenize(snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)

       UpperCAmelCase_				:  Dict  = tokenizer.encode(snake_case_ ,						add_special_tokens=snake_case_		)
       UpperCAmelCase_				:  Dict  = rust_tokenizer.encode(snake_case_ ,						add_special_tokens=snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)

       UpperCAmelCase_				:  str  = self.get_rust_tokenizer()
       UpperCAmelCase_				:  Union[str, Any]  = tokenizer.encode(snake_case_		)
       UpperCAmelCase_				:  List[str]  = rust_tokenizer.encode(snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)

       # With lower casing
       UpperCAmelCase_				:  Tuple  = self.get_tokenizer(do_lower_case=snake_case_		)
       UpperCAmelCase_				:  Union[str, Any]  = self.get_rust_tokenizer(do_lower_case=snake_case_		)

       UpperCAmelCase_				:  Tuple  = 'UNwant\u00E9d,running'

       UpperCAmelCase_				:  List[str]  = tokenizer.tokenize(snake_case_		)
       UpperCAmelCase_				:  List[str]  = rust_tokenizer.tokenize(snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)

       UpperCAmelCase_				:  Optional[int]  = tokenizer.encode(snake_case_ ,						add_special_tokens=snake_case_		)
       UpperCAmelCase_				:  Dict  = rust_tokenizer.encode(snake_case_ ,						add_special_tokens=snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)

       UpperCAmelCase_				:  Dict  = self.get_rust_tokenizer()
       UpperCAmelCase_				:  Tuple  = tokenizer.encode(snake_case_		)
       UpperCAmelCase_				:  Dict  = rust_tokenizer.encode(snake_case_		)
       self.assertListEqual(snake_case_ ,						snake_case_		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'		) ,						['ah', '\u535A', '\u63A8', 'zz']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = BasicTokenizer(do_lower_case=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		) ,						['hello', '!', 'how', 'are', 'you', '?']		)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		) ,						['hello']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = BasicTokenizer(do_lower_case=snake_case_ ,						strip_accents=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		) ,						['hällo', '!', 'how', 'are', 'you', '?']		)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		) ,						['h\u00E9llo']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = BasicTokenizer(do_lower_case=snake_case_ ,						strip_accents=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		) ,						['hallo', '!', 'how', 'are', 'you', '?']		)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		) ,						['hello']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = BasicTokenizer(do_lower_case=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		) ,						['hallo', '!', 'how', 'are', 'you', '?']		)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		) ,						['hello']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[str]  = BasicTokenizer(do_lower_case=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		) ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[str]  = BasicTokenizer(do_lower_case=snake_case_ ,						strip_accents=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		) ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Dict  = BasicTokenizer(do_lower_case=snake_case_ ,						strip_accents=snake_case_		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		) ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = BasicTokenizer(do_lower_case=snake_case_ ,						never_split=['[UNK]']		)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'		) ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       UpperCAmelCase_				:  Optional[Any]  = {}
       for i, token in enumerate(snake_case_		):
            UpperCAmelCase_				:  List[str]  = i
       UpperCAmelCase_				:  Union[str, Any]  = WordpieceTokenizer(vocab=snake_case_ ,						unk_token='[UNK]'		)

       self.assertListEqual(tokenizer.tokenize(''		) ,						[]		)

       self.assertListEqual(tokenizer.tokenize('unwanted running'		) ,						['un', '##want', '##ed', 'runn', '##ing']		)

       self.assertListEqual(tokenizer.tokenize('unwantedX running'		) ,						['[UNK]', 'runn', '##ing']		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       self.assertTrue(_is_whitespace(' '		)		)
       self.assertTrue(_is_whitespace('\t'		)		)
       self.assertTrue(_is_whitespace('\r'		)		)
       self.assertTrue(_is_whitespace('\n'		)		)
       self.assertTrue(_is_whitespace('\u00A0'		)		)

       self.assertFalse(_is_whitespace('A'		)		)
       self.assertFalse(_is_whitespace('-'		)		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       self.assertTrue(_is_control('\u0005'		)		)

       self.assertFalse(_is_control('A'		)		)
       self.assertFalse(_is_control(' '		)		)
       self.assertFalse(_is_control('\t'		)		)
       self.assertFalse(_is_control('\r'		)		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       self.assertTrue(_is_punctuation('-'		)		)
       self.assertTrue(_is_punctuation('$'		)		)
       self.assertTrue(_is_punctuation('`'		)		)
       self.assertTrue(_is_punctuation('.'		)		)

       self.assertFalse(_is_punctuation('A'		)		)
       self.assertFalse(_is_punctuation(' '		)		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(snake_case_		) for t in ['Test', '\xad', 'test']] ,						[['[UNK]'], [], ['[UNK]']]		)

       if self.test_rust_tokenizer:
            UpperCAmelCase_				:  Union[str, Any]  = self.get_rust_tokenizer()
            self.assertListEqual(
                [rust_tokenizer.tokenize(snake_case_		) for t in ['Test', '\xad', 'test']] ,						[['[UNK]'], [], ['[UNK]']]		)


  @slow
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = self.tokenizer_class.from_pretrained('bert-base-uncased'		)

       UpperCAmelCase_				:  Optional[int]  = tokenizer.encode('sequence builders' ,						add_special_tokens=snake_case_		)
       UpperCAmelCase_				:  Optional[int]  = tokenizer.encode('multi-sequence build' ,						add_special_tokens=snake_case_		)

       UpperCAmelCase_				:  Any  = tokenizer.build_inputs_with_special_tokens(snake_case_		)
       UpperCAmelCase_				:  Tuple  = tokenizer.build_inputs_with_special_tokens(snake_case_ ,						snake_case_		)

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                 UpperCAmelCase_				:  List[str]  = self.rust_tokenizer_class.from_pretrained(snake_case_ ,						**snake_case_		)

                 UpperCAmelCase_				:  Optional[int]  = F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                 UpperCAmelCase_				:  int  = tokenizer_r.encode_plus(
                     snake_case_ ,						return_attention_mask=snake_case_ ,						return_token_type_ids=snake_case_ ,						return_offsets_mapping=snake_case_ ,						add_special_tokens=snake_case_ ,						)

                 UpperCAmelCase_				:  Tuple  = tokenizer_r.do_lower_case if hasattr(snake_case_ ,						'do_lower_case'		) else False
                 UpperCAmelCase_				:  Optional[int]  = (
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'A'),
                         ((1, 2), ','),
                         ((3, 5), 'na'),
                         ((5, 6), '##ï'),
                         ((6, 8), '##ve'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'Allen'),
                         ((2_1, 2_3), '##NL'),
                         ((2_3, 2_4), '##P'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'a'),
                         ((1, 2), ','),
                         ((3, 8), 'naive'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'allen'),
                         ((2_1, 2_3), '##nl'),
                         ((2_3, 2_4), '##p'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results] ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']		)		)
                 self.assertEqual([e[0] for e in expected_results] ,						tokens['offset_mapping']		)






  @slow
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                 UpperCAmelCase_				:  Optional[int]  = self.rust_tokenizer_class.from_pretrained(snake_case_ ,						**snake_case_		)
                 UpperCAmelCase_				:  List[str]  = self.tokenizer_class.from_pretrained(snake_case_ ,						**snake_case_		)
                 UpperCAmelCase_				:  Optional[int]  = [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                 UpperCAmelCase_				:  str  = tokenizer_r.batch_encode_candidates(snake_case_ ,						max_length=1_0 ,						return_tensors='np'		)
                 UpperCAmelCase_				:  Tuple  = tokenizer_p.batch_encode_candidates(snake_case_ ,						max_length=1_0 ,						return_tensors='np'		)

                 UpperCAmelCase_				:  Optional[int]  = (2, 2, 1_0)

                 self.assertEqual(encoded_sentence_r['input_ids'].shape ,						snake_case_		)
                 self.assertEqual(encoded_sentence_r['attention_mask'].shape ,						snake_case_		)
                 self.assertEqual(encoded_sentence_r['token_type_ids'].shape ,						snake_case_		)

                 self.assertEqual(encoded_sentence_p['input_ids'].shape ,						snake_case_		)
                 self.assertEqual(encoded_sentence_p['attention_mask'].shape ,						snake_case_		)
                 self.assertEqual(encoded_sentence_p['token_type_ids'].shape ,						snake_case_		)







'''simple docstring'''


import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata


def    _lowerCamelCase       ( lowerCamelCase_   :				str  ):



     """simple docstring"""





     UpperCAmelCase_				:  str  = min(re.search('^ +'      ,	lowerCamelCase_  ).end() if t.startswith(' '  ) else 0 for t in string.splitlines()  )
     return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(lowerCamelCase_  )]  )


snake_case__	:     Dict      =    '''\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
'''

snake_case__	:     Union[str, Any]      =    '''\
---
---
# Begin of markdown

Some cool dataset card
'''


snake_case__	:     int      =    '''\
# Begin of markdown

Some cool dataset card
'''






class 	__SCREAMING_SNAKE_CASE		(			unittest.TestCase			):







  '''simple docstring'''






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmp_dir:
            UpperCAmelCase_				:  Tuple  = Path(snake_case_		) / 'README.md'
            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  Optional[Any]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertDictEqual(snake_case_ ,						{'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}		)

            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  Union[str, Any]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertDictEqual(snake_case_ ,						{}		)

            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  List[str]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertEqual(snake_case_ ,						{}		)






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '		)
       assert DatasetMetadata.from_yaml_string(snake_case_		)

       UpperCAmelCase_				:  int  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '		)
       with self.assertRaises(snake_case_		):
            DatasetMetadata.from_yaml_string(snake_case_		)

       UpperCAmelCase_				:  str  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '		)
       assert DatasetMetadata.from_yaml_string(snake_case_		)






