'''simple docstring'''




from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

__UpperCAmelCase     							=_symbol_database.Default()


__UpperCAmelCase     							=_descriptor_pool.Default().AddSerializedFile(
    b"\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"
)

__UpperCAmelCase     							=globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "sentencepiece_model_pb2", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
  __UpperCAmelCase     							=None
  __UpperCAmelCase     							=b"H\003"
  # (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
  # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
  # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
  # _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
  # _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
  __UpperCAmelCase     							=4_5
  __UpperCAmelCase     							=1_5_8_1
  __UpperCAmelCase     							=1_5_1_7
  __UpperCAmelCase     							=1_5_7_0
  __UpperCAmelCase     							=1_5_8_4
  __UpperCAmelCase     							=1_7_9_3
  __UpperCAmelCase     							=1_7_9_5
  __UpperCAmelCase     							=1_9_1_6
  __UpperCAmelCase     							=1_8_6_4
  __UpperCAmelCase     							=1_9_0_5
  __UpperCAmelCase     							=1_9_1_9
  __UpperCAmelCase     							=2_4_2_9
  __UpperCAmelCase     							=2_2_0_8
  __UpperCAmelCase     							=2_4_1_8
  __UpperCAmelCase     							=2_3_2_3
  __UpperCAmelCase     							=2_4_0_7
# @@protoc_insertion_point(module_scope)


import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


UpperCAmelCase	      = WebClient(token=os.environ['''CI_SLACK_BOT_TOKEN'''])






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    test_results.split(' '							)

	lowercase       =    0
	lowercase       =    0

	# When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
	# When it is too long, those signs are not present.
	lowercase       =    expressions[-2] if '=' in expressions[-1] else expressions[-1]

	for i, expression in enumerate(__SCREAMING_SNAKE_CASE							):
		if "failed" in expression:
			failed += int(expressions[i - 1]							)
		if "passed" in expression:
			success += int(expressions[i - 1]							)

	return failed, success, time_spent






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    {}
	lowercase       =    None
	lowercase       =    False
	for line in failures_short_lines.split('\n'							):
		if re.search(r'_ \[doctest\]' ,							__SCREAMING_SNAKE_CASE							):
			lowercase       =    True
			lowercase       =    line.split(' '							)[2]
		elif in_error and not line.split(' '							)[0].isdigit():
			lowercase       =    line
			lowercase       =    False

	return failures







class        A_   :




	'''simple docstring'''


	def __init__(       self		,    snake_case		,    snake_case       ):
		lowercase       =    title

		lowercase       =    doc_test_results['time_spent'].split(','       )[0]
		lowercase       =    doc_test_results['success']
		lowercase       =    doc_test_results['failures']
		lowercase       =    self.n_success + self.n_failures

		# Failures and success of the modeling tests
		lowercase       =    doc_test_results





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    [self._time_spent]
		lowercase       =    0

		for time in time_spent:
			lowercase       =    time.split(':'       )

			# Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
			if len(snake_case       ) == 1:
				lowercase       =    [0, 0, time_parts[0]]

			lowercase  ,					lowercase  ,					lowercase       =    int(time_parts[0]       ), int(time_parts[1]       ), float(time_parts[2]       )
			total_secs += hours * 3600 + minutes * 60 + seconds

		lowercase  ,					lowercase  ,					lowercase       =    total_secs // 3600, (total_secs % 3600) // 60, total_secs % 60
		return F'''{int(snake_case       )}h{int(snake_case       )}m{int(snake_case       )}s'''





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return {"type": "header", "text": {"type": "plain_text", "text": self.title}}





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return {
		    "type": "section",
		    "text": {
		        "type": "plain_text",
		        "text": F'''🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.''',
		        "emoji": True,
		    },
		    "accessory": {
		        "type": "button",
		        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
		        "url": F'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
		    },
		}





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return {
		    "type": "section",
		    "text": {
		        "type": "plain_text",
		        "text": (
		            F'''There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in'''
		            F''' {self.time}.'''
		        ),
		        "emoji": True,
		    },
		    "accessory": {
		        "type": "button",
		        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
		        "url": F'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
		    },
		}





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    40
		lowercase       =    {k: v['failed'] for k, v in doc_test_results.items() if isinstance(snake_case		,    snake_case       )}

		lowercase       =    ''
		for category, failures in category_failures.items():
			if len(snake_case       ) == 0:
				continue

			if report != "":
				report += "\n\n"

			report += F'''*{category} failures*:'''.ljust(line_length // 2       ).rjust(line_length // 2       ) + "\n"
			report += "`"
			report += "`\n`".join(snake_case       )
			report += "`"

		return {
		    "type": "section",
		    "text": {
		        "type": "mrkdwn",
		        "text": F'''The following examples had failures:\n\n\n{report}\n''',
		    },
		}





	@property
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    [self.header]

		if self.n_failures > 0:
			blocks.append(self.failures       )

		if self.n_failures > 0:
			blocks.extend([self.category_failures]       )

		if self.n_failures == 0:
			blocks.append(self.no_failures       )

		return json.dumps(snake_case       )





	@staticmethod
	def 	SCREAMING_SNAKE_CASE__						(       ):
		lowercase       =    [
		    {
		        'type': 'section',
		        'text': {
		            'type': 'plain_text',
		            'text': 'There was an issue running the tests.',
		        },
		        'accessory': {
		            'type': 'button',
		            'text': {'type': 'plain_text', 'text': 'Check Action results', 'emoji': True},
		            'url': F'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
		        },
		    }
		]

		print('Sending the following payload'       )
		print(json.dumps({'blocks': json.loads(snake_case       )}       )       )

		client.chat_postMessage(
		    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']		,    text='There was an issue running the tests.'		,    blocks=snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		print('Sending the following payload'       )
		print(json.dumps({'blocks': json.loads(self.payload       )}       )       )

		lowercase       =    F'''{self.n_failures} failures out of {self.n_tests} tests,''' if self.n_failures else 'All tests passed.'

		lowercase       =    client.chat_postMessage(
		    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']		,    blocks=self.payload		,    text=snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    snake_case       ):
		lowercase       =    ''
		for key, value in failures.items():
			lowercase       =    value[:200] + ' [Truncated]' if len(snake_case       ) > 250 else value
			failures_text += F'''*{key}*\n_{value}_\n\n'''

		lowercase       =    job_name
		lowercase       =    {'type': 'section', 'text': {'type': 'mrkdwn', 'text': text}}

		if job_link is not None:
			lowercase       =    {
			    'type': 'button',
			    'text': {'type': 'plain_text', 'text': 'GitHub Action job', 'emoji': True},
			    'url': job_link,
			}

		return [
		    {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
		    content,
		    {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
		]





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		if self.thread_ts is None:
			raise ValueError('Can only post reply if a post has been made.'       )

		lowercase       =    self.doc_test_results.pop('job_link'       )
		self.doc_test_results.pop('failures'       )
		self.doc_test_results.pop('success'       )
		self.doc_test_results.pop('time_spent'       )

		lowercase       =    sorted(self.doc_test_results.items()		,    key=lambda snake_case       : t[0]       )
		for job, job_result in sorted_dict:
			if len(job_result['failures']       ):
				lowercase       =    F'''*Num failures* :{len(job_result['failed']       )} \n'''
				lowercase       =    job_result['failures']
				lowercase       =    self.get_reply_blocks(snake_case		,    snake_case		,    snake_case		,    text=snake_case       )

				print('Sending the following reply'       )
				print(json.dumps({'blocks': blocks}       )       )

				client.chat_postMessage(
				    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']		,    text=F'''Results for {job}'''		,    blocks=snake_case		,    thread_ts=self.thread_ts['ts']		,    )

				time.sleep(1       )






def 							UpperCAmelCase_		(							):
	lowercase       =    os.environ['GITHUB_RUN_ID']
	lowercase       =    F'''https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100'''
	lowercase       =    requests.get(__SCREAMING_SNAKE_CASE							).json()
	lowercase       =    {}

	try:
		jobs.update({job['name']: job['html_url'] for job in result['jobs']}							)
		lowercase       =    math.ceil((result['total_count'] - 100) / 100							)

		for i in range(__SCREAMING_SNAKE_CASE							):
			lowercase       =    requests.get(url + F'''&page={i + 2}'''							).json()
			jobs.update({job['name']: job['html_url'] for job in result['jobs']}							)

		return jobs
	except Exception as e:
		print('Unknown error, could not fetch links.' ,							__SCREAMING_SNAKE_CASE							)

	return {}






def 							UpperCAmelCase_		(							__SCREAMING_SNAKE_CASE							):
	lowercase       =    {}

	if os.path.exists(__SCREAMING_SNAKE_CASE							):
		lowercase       =    os.listdir(__SCREAMING_SNAKE_CASE							)
		for file in files:
			try:
				with open(os.path.join(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							) ,							encoding='utf-8'							) as f:
					lowercase       =    f.read()
			except UnicodeDecodeError as e:
				raise ValueError(F'''Could not open {os.path.join(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)}.'''							) from e

	return _artifact






def 							UpperCAmelCase_		(							):







	class        A_   :




		'''simple docstring'''


		def __init__(       self		,    snake_case       ):
			lowercase       =    name
			lowercase       =    []





		def __str__(       self       ):
			return self.name





		def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
			self.paths.append({'name': self.name, 'path': path}       )




	lowercase       =    {}

	lowercase       =    filter(os.path.isdir ,							os.listdir()							)
	for directory in directories:
		lowercase       =    directory
		if artifact_name not in _available_artifacts:
			lowercase       =    Artifact(__SCREAMING_SNAKE_CASE							)

			_available_artifacts[artifact_name].add_path(__SCREAMING_SNAKE_CASE							)

	return _available_artifacts


if __name__ == "__main__":
	UpperCAmelCase	      = get_job_links()
	UpperCAmelCase	      = retrieve_available_artifacts()

	UpperCAmelCase	      = collections.OrderedDict(
	    [
	        ('''*.py''', '''API Examples'''),
	        ('''*.md''', '''MD Examples'''),
	    ]
	)

	# This dict will contain all the information relative to each doc test category:
	# - failed: list of failed tests
	# - failures: dict in the format 'test': 'error_message'
	UpperCAmelCase	      = {
	    v: {
	        '''failed''': [],
	        '''failures''': {},
	    }
	    for v in docs.values()
	}

	# Link to the GitHub Action job
	UpperCAmelCase	      = github_actions_job_links.get('''run_doctests''')

	UpperCAmelCase	      = available_artifacts['''doc_tests_gpu_test_reports'''].paths[0]
	UpperCAmelCase	      = retrieve_artifact(artifact_path['''name'''])
	if "stats" in artifact:
		UpperCAmelCase      , UpperCAmelCase      , UpperCAmelCase	      = handle_test_results(artifact['''stats'''])
		UpperCAmelCase	      = failed
		UpperCAmelCase	      = success
		UpperCAmelCase	      = time_spent[1:-1] + ''', '''

		UpperCAmelCase	      = extract_first_line_failure(artifact['''failures_short'''])
		for line in artifact["summary_short"].split('''\n'''):
			if re.search('''FAILED''', line):
				UpperCAmelCase	      = line.replace('''FAILED ''', '''''')
				UpperCAmelCase	      = line.split()[0].replace('''\n''', '''''')

				if "::" in line:
					UpperCAmelCase      , UpperCAmelCase	      = line.split('''::''')
				else:
					UpperCAmelCase      , UpperCAmelCase	      = line, line

				for file_regex in docs.keys():
					if fnmatch(file_path, file_regex):
						UpperCAmelCase	      = docs[file_regex]
						doc_test_results[category]["failed"].append(test)

						UpperCAmelCase	      = all_failures[test] if test in all_failures else '''N/A'''
						UpperCAmelCase	      = failure
						break

	UpperCAmelCase	      = Message('''🤗 Results of the doc tests.''', doc_test_results)
	message.post()
	message.post_reply()

