






import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

A__	:			Optional[int]			   = yaml.safe_load(
    '\\nname: ""\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: "Dataset Card for X" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: "Table of Contents"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: "Dataset Description"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: "Dataset Summary"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: "Supported Tasks and Leaderboards"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n'
)


A__	:			Optional[int]			   = {
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}


A__	:			Optional[Any]			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'


A__	:			Tuple			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			Any			   = {
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [
                                {
                                    'name': 'Extra Ignored Subsection',
                                    'text': '',
                                    'is_empty_text': True,
                                    'subsections': [],
                                }
                            ],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}

A__	:			List[Any]			   = '\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			int			   = (
    'The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README.'
)

A__	:			Optional[int]			   = '\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			int			   = (
    'The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README.'
)

A__	:			List[str]			   = '\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			Optional[Any]			   = 'The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README.'

A__	:			Any			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
A__	:			Optional[Any]			   = 'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored).'


A__	:			List[str]			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n'
A__	:			Dict			   = 'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found \'None\'.'

A__	:			str			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n'

A__	:			List[Any]			   = 'The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`.'


A__	:			List[Any]			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n'

A__	:			List[str]			   = 'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty.'

A__	:			Tuple			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
A__	:			str			   = 'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.'

A__	:			Tuple			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n'

A__	:			List[Any]			   = 'The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README.'

A__	:			Optional[Any]			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			Any			   = 'The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'

A__	:			Dict			   = ''

A__	:			List[Any]			   = 'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README.'

A__	:			Dict			   = '\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

A__	:			List[str]			   = 'The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections.'






@pytest.mark.parametrize(
    '''readme_md, expected_dict'''		,   [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				assert ReadMe.from_string(lowerCamelCase_		,   lowerCamelCase_				).to_dict() == expected_dict






@pytest.mark.parametrize(
    '''readme_md, expected_error'''		,   [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				with pytest.raises(lowerCamelCase_		,   match=re.escape(expected_error.format(path='''root'''				)				)				):
								lowercase__    =			ReadMe.from_string(lowerCamelCase_		,   lowerCamelCase_				)
								readme.validate()






@pytest.mark.parametrize(
    '''readme_md, expected_error'''		,   [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				with pytest.raises(lowerCamelCase_		,   match=re.escape(expected_error.format(path='''root'''				)				)				):
								ReadMe.from_string(lowerCamelCase_		,   lowerCamelCase_				)






@pytest.mark.parametrize(
    '''readme_md,'''		,   [
        (README_MULTIPLE_SAME_HEADING_1),
    ]		,   )
def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				ReadMe.from_string(lowerCamelCase_		,   lowerCamelCase_		,   suppress_parsing_errors=lowerCamelCase_				)






@pytest.mark.parametrize(
    '''readme_md, expected_dict'''		,   [
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								lowercase__    =			Path(lowerCamelCase_				) / '''README.md'''
								with open(lowerCamelCase_		,   '''w+'''				) as readme_file:
												readme_file.write(lowerCamelCase_				)
								lowercase__    =			ReadMe.from_readme(lowerCamelCase_		,   lowerCamelCase_				).to_dict()
								assert out["name"] == path
								assert out["text"] == ""
								assert out["is_empty_text"]
								assert out["subsections"] == expected_dict["subsections"]






@pytest.mark.parametrize(
    '''readme_md, expected_error'''		,   [
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								lowercase__    =			Path(lowerCamelCase_				) / '''README.md'''
								with open(lowerCamelCase_		,   '''w+'''				) as readme_file:
												readme_file.write(lowerCamelCase_				)
								lowercase__    =			expected_error.format(path=lowerCamelCase_				)
								with pytest.raises(lowerCamelCase_		,   match=re.escape(lowerCamelCase_				)				):
												lowercase__    =			ReadMe.from_readme(lowerCamelCase_		,   lowerCamelCase_				)
												readme.validate()






@pytest.mark.parametrize(
    '''readme_md, expected_error'''		,   [
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]		,   )
def   a			(						lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								lowercase__    =			Path(lowerCamelCase_				) / '''README.md'''
								with open(lowerCamelCase_		,   '''w+'''				) as readme_file:
												readme_file.write(lowerCamelCase_				)
								lowercase__    =			expected_error.format(path=lowerCamelCase_				)
								with pytest.raises(lowerCamelCase_		,   match=re.escape(lowerCamelCase_				)				):
												ReadMe.from_readme(lowerCamelCase_		,   lowerCamelCase_				)






@pytest.mark.parametrize(
    '''readme_md,'''		,   [
        (README_MULTIPLE_SAME_HEADING_1),
    ]		,   )
def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				with tempfile.TemporaryDirectory() as tmp_dir:
								lowercase__    =			Path(lowerCamelCase_				) / '''README.md'''
								with open(lowerCamelCase_		,   '''w+'''				) as readme_file:
												readme_file.write(lowerCamelCase_				)
								ReadMe.from_readme(lowerCamelCase_		,   lowerCamelCase_		,   suppress_parsing_errors=lowerCamelCase_				)












import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class       _UpperCAmelCase					(       A__		,unittest.TestCase	):

				"""simple docstring"""




				lowercase__											=       CTRLTokenizer
				lowercase__											=       False
				lowercase__											=       False




				def 				lowercase__   (     self						:    Union[str, Any]  ):

								'''simple docstring'''
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								lowercase__    =			['''adapt''', '''re@@''', '''a@@''', '''apt''', '''c@@''', '''t''', '''<unk>''']
								lowercase__    =			dict(zip(lowerCamelCase,  range(len(lowerCamelCase  )  )  )  )
								lowercase__    =			['''#version: 0.2''', '''a p''', '''ap t</w>''', '''r e''', '''a d''', '''ad apt</w>''', '''''']
								lowercase__    =			{'''unk_token''': '''<unk>'''}

								lowercase__    =			os.path.join(self.tmpdirname,  VOCAB_FILES_NAMES['''vocab_file''']  )
								lowercase__    =			os.path.join(self.tmpdirname,  VOCAB_FILES_NAMES['''merges_file''']  )
								with open(self.vocab_file,  '''w''',  encoding='''utf-8'''  ) as fp:
												fp.write(json.dumps(lowerCamelCase  ) + '''\n'''  )
								with open(self.merges_file,  '''w''',  encoding='''utf-8'''  ) as fp:
												fp.write('''\n'''.join(lowerCamelCase  )  )




				def 				lowercase__   (     self						:    Union[str, Any],  **lowerCamelCase						:    Dict  ):

								'''simple docstring'''
								kwargs.update(self.special_tokens_map  )
								return CTRLTokenizer.from_pretrained(self.tmpdirname,  **lowerCamelCase  )




				def 				lowercase__   (     self						:    Dict,  lowerCamelCase						:    Optional[int]  ):

								'''simple docstring'''
								lowercase__    =			'''adapt react readapt apt'''
								lowercase__    =			'''adapt react readapt apt'''
								return input_text, output_text




				def 				lowercase__   (     self						:    Tuple  ):

								'''simple docstring'''
								lowercase__    =			CTRLTokenizer(self.vocab_file,  self.merges_file,  **self.special_tokens_map  )
								lowercase__    =			'''adapt react readapt apt'''
								lowercase__    =			'''adapt re@@ a@@ c@@ t re@@ adapt apt'''.split()
								lowercase__    =			tokenizer.tokenize(lowerCamelCase  )
								self.assertListEqual(lowerCamelCase,  lowerCamelCase  )

								lowercase__    =			tokens + [tokenizer.unk_token]

								lowercase__    =			[0, 1, 2, 4, 5, 1, 0, 3, 6]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase  ),  lowerCamelCase  )




