






'''simple docstring'''





import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
		import importlib.resources as pkg_resources
except ImportError:
		# Try backported to PY<37 `importlib_resources`.
		import importlib_resources as pkg_resources

from . import resources


lowerCamelCase_		  =						"""https://github.com/huggingface/datasets/tree/main/src/datasets/utils"""
lowerCamelCase_		  =						f'''{BASE_REF_URL}/{__file__}'''
lowerCamelCase_		  =						logging.getLogger(__name__)
def   lowerCamelCase      (		a_			)			->    Tuple[Any, str]:
							lowerCAmelCase_       		=  pkg_resources.read_text(a_				,		a_			)
							return yaml.safe_load(a_			), F'''{BASE_REF_URL}/resources/{resource}'''


lowerCamelCase_  ,  lowerCamelCase_		  =						load_yaml_resource("""readme_structure.yaml""")

lowerCamelCase_		  =						[
    """[Needs More Information]""",
    """[More Information Needed]""",
    """(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)""",
]

# Dictionary representation of section/readme, error_list, warning_list
lowerCamelCase_		  =						Tuple[dict, List[str], List[str]]




class 							a_				:


							'''simple docstring'''






							def __init__(	self    ,    lowercase_    ,    lowercase_    ,    lowercase_ = None    ,    lowercase_ = False     )      ->      str:
														'''simple docstring'''

														lowerCAmelCase_       		=  name
														lowerCAmelCase_       		=  level
														lowerCAmelCase_       		=  lines
														lowerCAmelCase_       		=  ''
														lowerCAmelCase_       		=  True
														lowerCAmelCase_       		=  {}
														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														if self.lines is not None:
																					self.parse(suppress_parsing_errors=lowercase_     )

							def   _lowercase (	self    ,    lowercase_ = False     )      ->      Tuple:
														'''simple docstring'''

														lowerCAmelCase_       		=  ''
														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  False
														for line in self.lines:
																					if line.strip(' \n'     ) == "":
																												continue
																					elif line.strip(' \n'     )[:3] == "```":
																												lowerCAmelCase_       		=  not code_start
																					elif line.split()[0] == self.level + "#" and not code_start:
																												if current_sub_level != "":
																																			lowerCAmelCase_       		=  Section(lowercase_    ,    self.level + '#'    ,    lowercase_     )
																																			lowerCAmelCase_       		=  []
																												else:
																																			if current_lines != []:
																																										self.text += "".join(lowercase_     ).strip()
																																										if self.text != "" and self.text not in FILLER_TEXT:
																																																	lowerCAmelCase_       		=  False
																																										lowerCAmelCase_       		=  []

																												lowerCAmelCase_       		=  ' '.join(line.split()[1:]     ).strip(' \n'     )
																					else:
																												current_lines.append(lowercase_     )
														else:
																					if current_sub_level != "":
																												if current_sub_level in self.content:
																																			self.parsing_error_list.append(
																																			    f'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''     )
																												lowerCAmelCase_       		=  Section(lowercase_    ,    self.level + '#'    ,    lowercase_     )
																					else:
																												if current_lines != []:
																																			self.text += "".join(lowercase_     ).strip()
																																			if self.text != "" and self.text not in FILLER_TEXT:
																																										lowerCAmelCase_       		=  False

														if self.level == "" and not suppress_parsing_errors:
																					if self.parsing_error_list != [] or self.parsing_warning_list != []:
																												lowerCAmelCase_       		=  lowerCAmelCase_       		=  '\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list     )
																												lowerCAmelCase_       		=  f'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
																												raise ValueError(lowercase_     )

							def   _lowercase (	self    ,    lowercase_     )      ->      ReadmeValidatorOutput:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														if structure["allow_empty"] is False:
																					# If content is expected
																					if self.is_empty_text and self.content == {}:
																												# If no content is found, mention it in the error_list
																												error_list.append(f'''Expected some content in section `{self.name}` but it is empty.'''     )

														if structure["allow_empty_text"] is False:
																					# If some text is expected
																					if self.is_empty_text:
																												# If no text is found, mention it in the error_list
																												error_list.append(
																												    f'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''     )
        # Subsections Validation
														if structure["subsections"] is not None:
																					# If subsections are expected
																					if self.content == {}:
																												# If no subsections are present
																												lowerCAmelCase_       		=  [subsection['name'] for subsection in structure['subsections']]
																												# Mention the expected values in the error_list
																												error_list.append(
																												    f'''Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]     )}. Found \'None\'.'''     )
																					else:
																												# If some subsections are present
																												lowerCAmelCase_       		=  [subsection['name'] for subsection in structure['subsections']]
																												lowerCAmelCase_       		=  False
																												for idx, name in enumerate(lowercase_     ):
																																			if name not in self.content:
																																										# If the expected subsection is not present
																																										error_list.append(f'''Section `{self.name}` is missing subsection: `{name}`.'''     )
																																										lowerCAmelCase_       		=  True
																																			else:
																																										# If the subsection is present, validate subsection, return the result
																																										# and concat the errors from subsection to section error_list

																																										# Skip sublevel validation if current level is `###`
																																										if self.level == "###":
																																																	continue
																																										else:
																																																	lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_       		=  self.content[name].validate(
																																																	    structure['subsections'][idx]     )
																																										error_list += subsec_error_list
																																										warning_list += subsec_warning_list

																												if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																																			for name in self.content:
																																										if name not in structure_names:
																																																	# If an extra subsection is present
																																																	warning_list.append(
																																																	    f'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''     )
														if error_list:
																					# If there are errors, do not return the dictionary as it is invalid
																					return {}, error_list, warning_list
														else:
																					return self.to_dict(), error_list, warning_list






							def   _lowercase (	self     )      ->      dict:
														'''simple docstring'''

														return {
														    "name": self.name,
														    "text": self.text,
														    "is_empty_text": self.is_empty_text,
														    "subsections": [value.to_dict() for value in self.content.values()],
														}




class 							a_				(		a_	):  # Level 0


							'''simple docstring'''






							def __init__(	self    ,    lowercase_    ,    lowercase_    ,    lowercase_ = None    ,    lowercase_ = False     )      ->      Any:
														'''simple docstring'''

														super().__init__(name=lowercase_    ,    level=''     )  # Not using lines here as we need to use a child class parse
														lowerCAmelCase_       		=  structure
														lowerCAmelCase_       		=  -2
														lowerCAmelCase_       		=  0
														lowerCAmelCase_       		=  lines
														if self.lines is not None:
																					self.parse(suppress_parsing_errors=lowercase_     )

							def   _lowercase (	self     )      ->      Any:
														'''simple docstring'''

														if self.structure is None:
																					lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_       		=  self._validate(lowercase_     )
														else:
																					lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_       		=  self._validate(self.structure     )
														if error_list != [] or warning_list != []:
																					lowerCAmelCase_       		=  '\n'.join(['-\t' + x for x in error_list + warning_list]     )
																					lowerCAmelCase_       		=  f'''The following issues were found for the README at `{self.name}`:\n''' + errors
																					raise ValueError(lowercase_     )

							@classmethod
							def   _lowercase (	cls    ,    lowercase_    ,    lowercase_ = None    ,    lowercase_ = False     )      ->      List[str]:
														'''simple docstring'''

														with open(lowercase_    ,    encoding='utf-8'     ) as f:
																					lowerCAmelCase_       		=  f.readlines()
														return cls(lowercase_    ,    lowercase_    ,    lowercase_    ,    suppress_parsing_errors=lowercase_     )

							@classmethod
							def   _lowercase (	cls    ,    lowercase_    ,    lowercase_ = None    ,    lowercase_ = "root"    ,    lowercase_ = False     )      ->      Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  string.split('\n'     )
														return cls(lowercase_    ,    lowercase_    ,    lowercase_    ,    suppress_parsing_errors=lowercase_     )

							def   _lowercase (	self    ,    lowercase_ = False     )      ->      Union[str, Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  0

														for line in self.lines:
																					self.yaml_tags_line_count += 1
																					if line.strip(' \n'     ) == "---":
																												self.tag_count += 1
																												if self.tag_count == 2:
																																			break
																					line_count += 1
														if self.tag_count == 2:
																					lowerCAmelCase_       		=  self.lines[line_count + 1 :]  # Get the last + 1 th item.
														else:
																					lowerCAmelCase_       		=  self.lines[self.tag_count :]
														super().parse(suppress_parsing_errors=lowercase_     )

							def __str__(	self     )      ->      List[str]:
														'''simple docstring'''

														return str(self.to_dict()     )






							def   _lowercase (	self    ,    lowercase_     )      ->      Optional[int]:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  []
														if self.yaml_tags_line_count == 0:
																					warning_list.append('Empty YAML markers are present in the README.'     )
														elif self.tag_count == 0:
																					warning_list.append('No YAML markers are present in the README.'     )
														elif self.tag_count == 1:
																					warning_list.append('Only the start of YAML tags present in the README.'     )
														# Check how many first level sections are present.
														lowerCAmelCase_       		=  len(self.content.keys()     )
														if num_first_level_keys > 1:
																					# If more than one, add to the error list, continue
																					error_list.append(
																					    f'''The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()     )]     )}. Only one heading is expected. Skipping further validation for this README.'''     )
														elif num_first_level_keys < 1:
																					# If less than one, append error.
																					error_list.append(
																					    'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'     )

														else:
																					# If one exactly
																					lowerCAmelCase_       		=  list(self.content.keys()     )[0]  # Get the key
																					if start_key.startswith('Dataset Card for'     ):  # Check correct start
																												# If the starting is correct, validate all the sections
																												lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_       		=  self.content[start_key].validate(
																												    readme_structure['subsections'][0]     )
																												error_list += sec_error_list
																												warning_list += sec_warning_list
																					else:
																												# If not found, append error
																												error_list.append(
																												    'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'     )
														if error_list:
																					# If there are errors, do not return the dictionary as it is invalid
																					return {}, error_list, warning_list
														else:
																					return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
		from argparse import ArgumentParser

		lowerCamelCase_		  =						ArgumentParser(usage="""Validate the content (excluding YAML tags) of a README.md file.""")
		ap.add_argument("""readme_filepath""")
		lowerCamelCase_		  =						ap.parse_args()
		lowerCamelCase_		  =						Path(args.readme_filepath)
		lowerCamelCase_		  =						ReadMe.from_readme(readme_filepath)











import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
		from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


lowerCamelCase_		  =						get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_torch
@require_torchaudio
@require_sentencepiece
class 							a_				(		unittest.TestCase	):


							'''simple docstring'''






							def   _lowercase (	self     )      ->      Optional[int]:
														'''simple docstring'''

														lowerCAmelCase_       		=  tempfile.mkdtemp()

														lowerCAmelCase_       		=  ['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
														lowerCAmelCase_       		=  dict(zip(lowercase_    ,    range(len(lowercase_     )     )     )     )
														lowerCAmelCase_       		=  Path(self.tmpdirname     )
														save_json(lowercase_    ,    save_dir / VOCAB_FILES_NAMES['vocab_file']     )
														if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																					copyfile(lowercase_    ,    save_dir / VOCAB_FILES_NAMES['spm_file']     )

														lowerCAmelCase_       		=  SpeechaTextTokenizer.from_pretrained(self.tmpdirname     )
														tokenizer.save_pretrained(self.tmpdirname     )

														lowerCAmelCase_       		=  {
														    'feature_size': 2_4,
														    'num_mel_bins': 2_4,
														    'padding_value': 0.0,
														    'sampling_rate': 1_6_0_0_0,
														    'return_attention_mask': False,
														    'do_normalize': True,
														}
														save_json(lowercase_    ,    save_dir / FEATURE_EXTRACTOR_NAME     )

							def   _lowercase (	self    ,    **lowercase_     )      ->      List[Any]:
														'''simple docstring'''

														return SpeechaTextTokenizer.from_pretrained(self.tmpdirname    ,    **lowercase_     )

							def   _lowercase (	self    ,    **lowercase_     )      ->      Optional[Any]:
														'''simple docstring'''

														return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname    ,    **lowercase_     )

							def   _lowercase (	self     )      ->      List[Any]:
														'''simple docstring'''

														shutil.rmtree(self.tmpdirname     )

							def   _lowercase (	self     )      ->      List[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.get_tokenizer()
														lowerCAmelCase_       		=  self.get_feature_extractor()

														lowerCAmelCase_       		=  SpeechaTextProcessor(tokenizer=lowercase_    ,    feature_extractor=lowercase_     )

														processor.save_pretrained(self.tmpdirname     )
														lowerCAmelCase_       		=  SpeechaTextProcessor.from_pretrained(self.tmpdirname     )

														self.assertEqual(processor.tokenizer.get_vocab()    ,    tokenizer.get_vocab()     )
														self.assertIsInstance(processor.tokenizer    ,    lowercase_     )

														self.assertEqual(processor.feature_extractor.to_json_string()    ,    feature_extractor.to_json_string()     )
														self.assertIsInstance(processor.feature_extractor    ,    lowercase_     )

							def   _lowercase (	self     )      ->      Union[str, Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  SpeechaTextProcessor(
														    tokenizer=self.get_tokenizer()    ,    feature_extractor=self.get_feature_extractor()     )
														processor.save_pretrained(self.tmpdirname     )

														lowerCAmelCase_       		=  self.get_tokenizer(bos_token='(BOS)'    ,    eos_token='(EOS)'     )
														lowerCAmelCase_       		=  self.get_feature_extractor(do_normalize=lowercase_    ,    padding_value=1.0     )

														lowerCAmelCase_       		=  SpeechaTextProcessor.from_pretrained(
														    self.tmpdirname    ,    bos_token='(BOS)'    ,    eos_token='(EOS)'    ,    do_normalize=lowercase_    ,    padding_value=1.0     )

														self.assertEqual(processor.tokenizer.get_vocab()    ,    tokenizer_add_kwargs.get_vocab()     )
														self.assertIsInstance(processor.tokenizer    ,    lowercase_     )

														self.assertEqual(processor.feature_extractor.to_json_string()    ,    feature_extractor_add_kwargs.to_json_string()     )
														self.assertIsInstance(processor.feature_extractor    ,    lowercase_     )

							def   _lowercase (	self     )      ->      Dict:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.get_feature_extractor()
														lowerCAmelCase_       		=  self.get_tokenizer()

														lowerCAmelCase_       		=  SpeechaTextProcessor(tokenizer=lowercase_    ,    feature_extractor=lowercase_     )

														lowerCAmelCase_       		=  floats_list((3, 1_0_0_0)     )

														lowerCAmelCase_       		=  feature_extractor(lowercase_    ,    return_tensors='np'     )
														lowerCAmelCase_       		=  processor(lowercase_    ,    return_tensors='np'     )

														for key in input_feat_extract.keys():
																					self.assertAlmostEqual(input_feat_extract[key].sum()    ,    input_processor[key].sum()    ,    delta=1e-2     )

							def   _lowercase (	self     )      ->      Union[str, Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.get_feature_extractor()
														lowerCAmelCase_       		=  self.get_tokenizer()

														lowerCAmelCase_       		=  SpeechaTextProcessor(tokenizer=lowercase_    ,    feature_extractor=lowercase_     )

														lowerCAmelCase_       		=  'This is a test string'

														lowerCAmelCase_       		=  processor(text=lowercase_     )

														lowerCAmelCase_       		=  tokenizer(lowercase_     )

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]    ,    encoded_processor[key]     )

							def   _lowercase (	self     )      ->      Any:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.get_feature_extractor()
														lowerCAmelCase_       		=  self.get_tokenizer()

														lowerCAmelCase_       		=  SpeechaTextProcessor(tokenizer=lowercase_    ,    feature_extractor=lowercase_     )

														lowerCAmelCase_       		=  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														lowerCAmelCase_       		=  processor.batch_decode(lowercase_     )
														lowerCAmelCase_       		=  tokenizer.batch_decode(lowercase_     )

														self.assertListEqual(lowercase_    ,    lowercase_     )






							def   _lowercase (	self     )      ->      Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.get_feature_extractor()
														lowerCAmelCase_       		=  self.get_tokenizer()

														lowerCAmelCase_       		=  SpeechaTextProcessor(tokenizer=lowercase_    ,    feature_extractor=lowercase_     )

														self.assertListEqual(
														    processor.model_input_names    ,    feature_extractor.model_input_names    ,    msg='`processor` and `feature_extractor` model input names do not match'    ,    )



