


"""simple docstring"""
import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE							:			Union[str, Any]						=			get_tests_dir('fixtures/test_sentencepiece.model')
__SCREAMING_SNAKE_CASE							:			Optional[Any]						=			get_tests_dir('fixtures/test_sentencepiece_bpe.model')

__SCREAMING_SNAKE_CASE							:			Tuple						=			'pt' if is_torch_available() else 'tf'





@require_sentencepiece
@require_tokenizers
class    __A   (__A  ,						unittest.TestCase):





							'''simple docstring'''

							__lowercase:						List[Any]												=			CamembertTokenizer
							__lowercase:						str												=			CamembertTokenizerFast
							__lowercase:						str												=			True
							__lowercase:						Any												=			True



							def 						lowerCAmelCase				( self							:					Tuple				)    ->Optional[Any]:



														"""simple docstring"""



														super().setUp()

														# We have a SentencePiece fixture for testing
														snake_case_							=						CamembertTokenizer(__UpperCAmelCase				)
														tokenizer.save_pretrained(self.tmpdirname				)



							def 						lowerCAmelCase				( self							:					List[Any]				)    ->List[str]:



														"""simple docstring"""



														snake_case_							=						"""<pad>"""
														snake_case_							=						1

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				)						,				__UpperCAmelCase				)
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				)						,				__UpperCAmelCase				)



							def 						lowerCAmelCase				( self							:					int				)    ->Dict:



														"""simple docstring"""



														snake_case_							=						list(self.get_tokenizer().get_vocab().keys()				)

														self.assertEqual(vocab_keys[0]						,				"""<s>NOTUSED"""				)
														self.assertEqual(vocab_keys[1]						,				"""<pad>"""				)
														self.assertEqual(vocab_keys[-1]						,				"""<mask>"""				)
														self.assertEqual(len(__UpperCAmelCase				)						,				1_004				)



							def 						lowerCAmelCase				( self							:					str				)    ->Tuple:



														"""simple docstring"""



														self.assertEqual(self.get_tokenizer().vocab_size						,				1_005				)



							def 						lowerCAmelCase				( self							:					Tuple				)    ->List[Any]:



														"""simple docstring"""



														snake_case_							=						CamembertTokenizer(__UpperCAmelCase				)
														tokenizer.save_pretrained(self.tmpdirname				)
														snake_case_							=						CamembertTokenizerFast.from_pretrained(self.tmpdirname				)

														snake_case_							=						"""I was born in 92000, and this is falsé."""

														snake_case_							=						tokenizer.encode(__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.encode(__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)

														snake_case_							=						tokenizer.encode(__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.encode(__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)

														# <unk> tokens are not the same for `rust` than for `slow`.
														# Because spm gives back raw token instead of `unk` in EncodeAsPieces
														# tokens = tokenizer.tokenize(sequence)
														snake_case_							=						tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.tokenize(__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)



							def 						lowerCAmelCase				( self							:					Optional[Any]				)    ->Dict:



														"""simple docstring"""



														if not self.test_rust_tokenizer:
																					return

														snake_case_							=						self.get_tokenizer()
														snake_case_							=						self.get_rust_tokenizer()

														snake_case_							=						"""I was born in 92000, and this is falsé."""

														snake_case_							=						tokenizer.tokenize(__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.tokenize(__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)

														snake_case_							=						tokenizer.encode(__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.encode(__UpperCAmelCase						,				add_special_tokens=__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)

														snake_case_							=						self.get_rust_tokenizer()
														snake_case_							=						tokenizer.encode(__UpperCAmelCase				)
														snake_case_							=						rust_tokenizer.encode(__UpperCAmelCase				)
														self.assertListEqual(__UpperCAmelCase						,				__UpperCAmelCase				)



							@slow
							def 						lowerCAmelCase				( self							:					Any				)    ->Dict:



														"""simple docstring"""



														snake_case_							=						{"""input_ids""": [[5, 54, 7_196, 297, 30, 23, 776, 18, 11, 3_215, 3_705, 8_252, 22, 3_164, 1_181, 2_116, 29, 16, 813, 25, 791, 3_314, 20, 3_446, 38, 27_575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9_088, 20, 1_517, 8, 22_804, 18_818, 10, 38, 629, 607, 607, 142, 19, 7_196, 867, 56, 10_326, 24, 2_267, 20, 416, 5_072, 15_612, 233, 734, 7, 2_399, 27, 16, 3_015, 1_649, 7, 24, 20, 4_338, 2_399, 27, 13, 3_400, 14, 13, 6_189, 8, 930, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
														# fmt: on

														# camembert is a french model. So we also use french texts.
														snake_case_							=						[
														    """Le transformeur est un modèle d\'apprentissage profond introduit en 2017, """
														    """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
														    """À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
														    """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
														    """telles que la traduction et la synthèse de texte.""",
														]

														self.tokenizer_integration_test_util(
														    expected_encoding=__UpperCAmelCase						,				model_name="""camembert-base"""						,				revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""						,				sequences=__UpperCAmelCase						,				)




"""simple docstring"""




import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


__lowerCamelCase							 =     WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"])






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							test_results.split(' ' )

							A__		   =							0
							A__		   =							0

							# When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
							# When it is too long, those signs are not present.
							A__		   =							expressions[-2] if '=' in expressions[-1] else expressions[-1]

							for i, expression in enumerate(UpperCamelCase__ ):
														if "failed" in expression:
																					failed += int(expressions[i - 1] )
														if "passed" in expression:
																					success += int(expressions[i - 1] )

							return failed, success, time_spent






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							{}
							A__		   =							None
							A__		   =							False
							for line in failures_short_lines.split('\n' ):
														if re.search(r'_ \[doctest\]'				,			UpperCamelCase__ ):
																					A__		   =							True
																					A__		   =							line.split(' ' )[2]
														elif in_error and not line.split(' ' )[0].isdigit():
																					A__		   =							line
																					A__		   =							False

							return failures
class 			UpperCamelCase__:





						def __init__(				self						,__UpperCAmelCase						,__UpperCAmelCase   )    -> Dict:
													A__		   =							title

													A__		   =							doc_test_results['time_spent'].split(','   )[0]
													A__		   =							doc_test_results['success']
													A__		   =							doc_test_results['failures']
													A__		   =							self.n_success + self.n_failures

													# Failures and success of the modeling tests
													A__		   =							doc_test_results





						@property
						def  snake_case__		(				self   )    -> str:
													A__		   =							[self._time_spent]
													A__		   =							0

													for time in time_spent:
																				A__		   =							time.split(':'   )

																				# Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
																				if len(__UpperCAmelCase   ) == 1:
																											A__		   =							[0, 0, time_parts[0]]

																				A__   ,     A__   ,     A__		   =							int(time_parts[0]   ), int(time_parts[1]   ), float(time_parts[2]   )
																				total_secs += hours * 36_00 + minutes * 60 + seconds

													A__   ,     A__   ,     A__		   =							total_secs // 36_00, (total_secs % 36_00) // 60, total_secs % 60
													return f'''{int(__UpperCAmelCase   )}h{int(__UpperCAmelCase   )}m{int(__UpperCAmelCase   )}s'''





						@property
						def  snake_case__		(				self   )    -> Dict:
													return {"type": "header", "text": {"type": "plain_text", "text": self.title}}





						@property
						def  snake_case__		(				self   )    -> Dict:
													return {
													    "type": "section",
													    "text": {
													        "type": "plain_text",
													        "text": f'''🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.''',
													        "emoji": True,
													    },
													    "accessory": {
													        "type": "button",
													        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
													        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}''',
													    },
													}





						@property
						def  snake_case__		(				self   )    -> Dict:
													return {
													    "type": "section",
													    "text": {
													        "type": "plain_text",
													        "text": (
													            f'''There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in'''
													            f''' {self.time}.'''
													        ),
													        "emoji": True,
													    },
													    "accessory": {
													        "type": "button",
													        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
													        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}''',
													    },
													}





						@property
						def  snake_case__		(				self   )    -> Dict:
													A__		   =							40
													A__		   =							{k: v['failed'] for k, v in doc_test_results.items() if isinstance(__UpperCAmelCase						,__UpperCAmelCase   )}

													A__		   =							''
													for category, failures in category_failures.items():
																				if len(__UpperCAmelCase   ) == 0:
																											continue

																				if report != "":
																											report += "\n\n"

																				report += f'''*{category} failures*:'''.ljust(line_length // 2   ).rjust(line_length // 2   ) + "\n"
																				report += "`"
																				report += "`\n`".join(__UpperCAmelCase   )
																				report += "`"

													return {
													    "type": "section",
													    "text": {
													        "type": "mrkdwn",
													        "text": f'''The following examples had failures:\n\n\n{report}\n''',
													    },
													}





						@property
						def  snake_case__		(				self   )    -> str:
													A__		   =							[self.header]

													if self.n_failures > 0:
																				blocks.append(self.failures   )

													if self.n_failures > 0:
																				blocks.extend([self.category_failures]   )

													if self.n_failures == 0:
																				blocks.append(self.no_failures   )

													return json.dumps(__UpperCAmelCase   )





						@staticmethod
						def  snake_case__		(				)    -> Any:
													A__		   =							[
													    {
													        'type': 'section',
													        'text': {
													            'type': 'plain_text',
													            'text': 'There was an issue running the tests.',
													        },
													        'accessory': {
													            'type': 'button',
													            'text': {'type': 'plain_text', 'text': 'Check Action results', 'emoji': True},
													            'url': f'''https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}''',
													        },
													    }
													]

													print('Sending the following payload'   )
													print(json.dumps({'blocks': json.loads(__UpperCAmelCase   )}   )   )

													client.chat_postMessage(
													    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']						,text='There was an issue running the tests.'						,blocks=__UpperCAmelCase						,)





						def  snake_case__		(				self   )    -> int:
													print('Sending the following payload'   )
													print(json.dumps({'blocks': json.loads(self.payload   )}   )   )

													A__		   =							f'''{self.n_failures} failures out of {self.n_tests} tests,''' if self.n_failures else 'All tests passed.'

													A__		   =							client.chat_postMessage(
													    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']						,blocks=self.payload						,text=__UpperCAmelCase						,)





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> int:
													A__		   =							''
													for key, value in failures.items():
																				A__		   =							value[:2_00] + ' [Truncated]' if len(__UpperCAmelCase   ) > 2_50 else value
																				failures_text += f'''*{key}*\n_{value}_\n\n'''

													A__		   =							job_name
													A__		   =							{'type': 'section', 'text': {'type': 'mrkdwn', 'text': text}}

													if job_link is not None:
																				A__		   =							{
																				    'type': 'button',
																				    'text': {'type': 'plain_text', 'text': 'GitHub Action job', 'emoji': True},
																				    'url': job_link,
																				}

													return [
													    {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
													    content,
													    {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
													]





						def  snake_case__		(				self   )    -> Dict:
													if self.thread_ts is None:
																				raise ValueError('Can only post reply if a post has been made.'   )

													A__		   =							self.doc_test_results.pop('job_link'   )
													self.doc_test_results.pop('failures'   )
													self.doc_test_results.pop('success'   )
													self.doc_test_results.pop('time_spent'   )

													A__		   =							sorted(self.doc_test_results.items()						,key=lambda __UpperCAmelCase   : t[0]   )
													for job, job_result in sorted_dict:
																				if len(job_result['failures']   ):
																											A__		   =							f'''*Num failures* :{len(job_result["failed"]   )} \n'''
																											A__		   =							job_result['failures']
																											A__		   =							self.get_reply_blocks(__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,text=__UpperCAmelCase   )

																											print('Sending the following reply'   )
																											print(json.dumps({'blocks': blocks}   )   )

																											client.chat_postMessage(
																											    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']						,text=f'''Results for {job}'''						,blocks=__UpperCAmelCase						,thread_ts=self.thread_ts['ts']						,)

																											time.sleep(1   )






def      UpperCAmelCase    (		):






							"""simple docstring"""





							A__		   =							os.environ['GITHUB_RUN_ID']
							A__		   =							F'''https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100'''
							A__		   =							requests.get(UpperCamelCase__ ).json()
							A__		   =							{}

							try:
														jobs.update({job['name']: job['html_url'] for job in result['jobs']} )
														A__		   =							math.ceil((result['total_count'] - 100) / 100 )

														for i in range(UpperCamelCase__ ):
																					A__		   =							requests.get(url + F'''&page={i + 2}''' ).json()
																					jobs.update({job['name']: job['html_url'] for job in result['jobs']} )

														return jobs
							except Exception as e:
														print('Unknown error, could not fetch links.'				,			UpperCamelCase__ )

							return {}






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							{}

							if os.path.exists(UpperCamelCase__ ):
														A__		   =							os.listdir(UpperCamelCase__ )
														for file in files:
																					try:
																												with open(os.path.join(UpperCamelCase__				,			UpperCamelCase__ )				,			encoding='utf-8' ) as f:
																																			A__		   =							f.read()
																					except UnicodeDecodeError as e:
																												raise ValueError(F'''Could not open {os.path.join(UpperCamelCase__				,			UpperCamelCase__ )}.''' ) from e

							return _artifact






def      UpperCAmelCase    (		):






							"""simple docstring"""





							class 			UpperCamelCase__:





													def __init__(				self						,__UpperCAmelCase   )    -> Optional[Any]:
																				A__		   =							name
																				A__		   =							[]





													def __str__(				self   )    -> List[str]:
																				return self.name





													def  snake_case__		(				self						,__UpperCAmelCase   )    -> int:
																				self.paths.append({'name': self.name, 'path': path}   )






							A__		   =							{}

							A__		   =							filter(os.path.isdir				,			os.listdir() )
							for directory in directories:
														A__		   =							directory
														if artifact_name not in _available_artifacts:
																					A__		   =							Artifact(UpperCamelCase__ )

																					_available_artifacts[artifact_name].add_path(UpperCamelCase__ )

							return _available_artifacts


if __name__ == "__main__":
				__lowerCamelCase							 =     get_job_links()
				__lowerCamelCase							 =     retrieve_available_artifacts()

				__lowerCamelCase							 =     collections.OrderedDict(
				    [
				        ("*.py", "API Examples"),
				        ("*.md", "MD Examples"),
				    ]
				)

				# This dict will contain all the information relative to each doc test category:
				# - failed: list of failed tests
				# - failures: dict in the format 'test': 'error_message'
				__lowerCamelCase							 =     {
				    v: {
				        "failed": [],
				        "failures": {},
				    }
				    for v in docs.values()
				}

				# Link to the GitHub Action job
				__lowerCamelCase							 =     github_actions_job_links.get("run_doctests")

				__lowerCamelCase							 =     available_artifacts["doc_tests_gpu_test_reports"].paths[0]
				__lowerCamelCase							 =     retrieve_artifact(artifact_path["name"])
				if "stats" in artifact:
								__lowerCamelCase ,      __lowerCamelCase ,      __lowerCamelCase							 =     handle_test_results(artifact["stats"])
								__lowerCamelCase							 =     failed
								__lowerCamelCase							 =     success
								__lowerCamelCase							 =     time_spent[1:-1] + ", "

								__lowerCamelCase							 =     extract_first_line_failure(artifact["failures_short"])
								for line in artifact["summary_short"].split("\n"):
												if re.search("FAILED", line):
																__lowerCamelCase							 =     line.replace("FAILED ", "")
																__lowerCamelCase							 =     line.split()[0].replace("\n", "")

																if "::" in line:
																				__lowerCamelCase ,      __lowerCamelCase							 =     line.split("::")
																else:
																				__lowerCamelCase ,      __lowerCamelCase							 =     line, line

																for file_regex in docs.keys():
																				if fnmatch(file_path, file_regex):
																								__lowerCamelCase							 =     docs[file_regex]
																								doc_test_results[category]["failed"].append(test)

																								__lowerCamelCase							 =     all_failures[test] if test in all_failures else "N/A"
																								__lowerCamelCase							 =     failure
																								break

				__lowerCamelCase							 =     Message("🤗 Results of the doc tests.", doc_test_results)
				message.post()
				message.post_reply()





