




import argparse
import ast
import logging
import os
import sys

import pandas as pd
import torch
from tqdm import tqdm

from transformers import BartForConditionalGeneration, RagRetriever, RagSequenceForGeneration, RagTokenForGeneration
from transformers import logging as transformers_logging


sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # isort:skip
from utils_rag import exact_match_score, fa_score  # noqa: E402 # isort:skip


lowercase__				:     Union[str, Any] 		=						logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

transformers_logging.set_verbosity_info()


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	if "token" in model_name_or_path:
		return "rag_token"
	if "sequence" in model_name_or_path:
		return "rag_sequence"
	if "bart" in model_name_or_path:
		return "bart"
	return None


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	return max(metric_fn(_A     ,	_A							) for gt in ground_truths							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     [line.strip() for line in open(_A     ,	"r"							).readlines()]
	snake_case_		    =     []

	if args.gold_data_mode == "qa":
		snake_case_		    =     pd.read_csv(_A     ,	sep="\t"     ,	header=_A							)
		for answer_list in data[1]:
			snake_case_		    =     ast.literal_eval(_A							)
			answers.append(_A							)
	else:
		snake_case_		    =     [line.strip() for line in open(_A     ,	"r"							).readlines()]
		snake_case_		    =     [[reference] for reference in references]

	snake_case_		    =     snake_case_		    =     snake_case_		    =     0
	for prediction, ground_truths in zip(_A     ,	_A							):
		total += 1
		em += metric_max_over_ground_truths(_A     ,	_A     ,	_A							)
		fa += metric_max_over_ground_truths(_A     ,	_A     ,	_A							)

	snake_case_		    =     1_00.0 * em / total
	snake_case_		    =     1_00.0 * fa / total

	logger.info(f"F1: {fa:.2f}"							)
	logger.info(f"EM: {em:.2f}"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     args.k
	snake_case_		    =     [line.strip() for line in open(_A     ,	"r"							).readlines()]
	snake_case_		    =     [line.strip() for line in open(_A     ,	"r"							).readlines()]

	snake_case_		    =     snake_case_		    =     0
	for hypo, reference in zip(_A     ,	_A							):
		snake_case_		    =     set(hypo.split("\t"							)[:k]							)
		snake_case_		    =     set(reference.split("\t"							)							)
		total += 1
		em += len(hypo_provenance & ref_provenance							) / k

	snake_case_		    =     1_00.0 * em / total
	logger.info(f"Precision@{k}: {em: .2f}"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	def strip_title(_A							):
		if title.startswith("\""							):
			snake_case_		    =     title[1:]
		if title.endswith("\""							):
			snake_case_		    =     title[:-1]
		return title

	snake_case_		    =     rag_model.retriever.question_encoder_tokenizer.batch_encode_plus(
	    _A     ,	return_tensors="pt"     ,	padding=_A     ,	truncation=_A     ,	)["input_ids"].to(args.device							)

	snake_case_		    =     rag_model.rag.question_encoder(_A							)
	snake_case_		    =     question_enc_outputs[0]

	snake_case_		    =     rag_model.retriever(
	    _A     ,	question_enc_pool_output.cpu().detach().to(torch.floataa							).numpy()     ,	prefix=rag_model.rag.generator.config.prefix     ,	n_docs=rag_model.config.n_docs     ,	return_tensors="pt"     ,	)
	snake_case_		    =     rag_model.retriever.index.get_doc_dicts(result.doc_ids							)
	snake_case_		    =     []
	for docs in all_docs:
		snake_case_		    =     [strip_title(_A							) for title in docs["title"]]
		provenance_strings.append("\t".join(_A							)							)
	return provenance_strings


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	with torch.no_grad():
		snake_case_		    =     rag_model.retriever.question_encoder_tokenizer.batch_encode_plus(
		    _A     ,	return_tensors="pt"     ,	padding=_A     ,	truncation=_A							)

		snake_case_		    =     inputs_dict.input_ids.to(args.device							)
		snake_case_		    =     inputs_dict.attention_mask.to(args.device							)
		snake_case_		    =     rag_model.generate(  # rag_model overwrites generate
		    _A     ,	attention_mask=_A     ,	num_beams=args.num_beams     ,	min_length=args.min_length     ,	max_length=args.max_length     ,	early_stopping=_A     ,	num_return_sequences=1     ,	bad_words_ids=[[0, 0]]     ,	)
		snake_case_		    =     rag_model.retriever.generator_tokenizer.batch_decode(_A     ,	skip_special_tokens=_A							)

		if args.print_predictions:
			for q, a in zip(_A     ,	_A							):
				logger.info("Q: {} - A: {}".format(_A     ,	_A							)							)

		return answers


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	snake_case_		    =     argparse.ArgumentParser()
	parser.add_argument(
	    "--model_type"     ,	choices=["rag_sequence", "rag_token", "bart"]     ,	type=_A     ,	help=(
	        "RAG model type: rag_sequence, rag_token or bart, if none specified, the type is inferred from the"
	        " model_name_or_path"
	    )     ,	)
	parser.add_argument(
	    "--index_name"     ,	default=_A     ,	choices=["exact", "compressed", "legacy"]     ,	type=_A     ,	help="RAG model retriever type"     ,	)
	parser.add_argument(
	    "--index_path"     ,	default=_A     ,	type=_A     ,	help="Path to the retrieval index"     ,	)
	parser.add_argument("--n_docs"     ,	default=5     ,	type=_A     ,	help="Number of retrieved docs"							)
	parser.add_argument(
	    "--model_name_or_path"     ,	default=_A     ,	type=_A     ,	required=_A     ,	help="Path to pretrained checkpoints or model identifier from huggingface.co/models"     ,	)
	parser.add_argument(
	    "--eval_mode"     ,	choices=["e2e", "retrieval"]     ,	default="e2e"     ,	type=_A     ,	help=(
	        "Evaluation mode, e2e calculates exact match and F1 of the downstream task, retrieval calculates"
	        " precision@k."
	    )     ,	)
	parser.add_argument("--k"     ,	default=1     ,	type=_A     ,	help="k for the precision@k calculation"							)
	parser.add_argument(
	    "--evaluation_set"     ,	default=_A     ,	type=_A     ,	required=_A     ,	help="Path to a file containing evaluation samples"     ,	)
	parser.add_argument(
	    "--gold_data_path"     ,	default=_A     ,	type=_A     ,	required=_A     ,	help="Path to a tab-separated file with gold samples"     ,	)
	parser.add_argument(
	    "--gold_data_mode"     ,	default="qa"     ,	type=_A     ,	choices=["qa", "ans"]     ,	help=(
	        "Format of the gold data file"
	        "qa - a single line in the following format: question [tab] answer_list"
	        "ans - a single line of the gold file contains the expected answer string"
	    )     ,	)
	parser.add_argument(
	    "--predictions_path"     ,	type=_A     ,	default="predictions.txt"     ,	help="Name of the predictions file, to be stored in the checkpoints directory"     ,	)
	parser.add_argument(
	    "--eval_all_checkpoints"     ,	action="store_true"     ,	help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number"     ,	)
	parser.add_argument(
	    "--eval_batch_size"     ,	default=8     ,	type=_A     ,	help="Batch size per GPU/CPU for evaluation."     ,	)
	parser.add_argument(
	    "--recalculate"     ,	help="Recalculate predictions even if the prediction file exists"     ,	action="store_true"     ,	)
	parser.add_argument(
	    "--num_beams"     ,	default=4     ,	type=_A     ,	help="Number of beams to be used when generating answers"     ,	)
	parser.add_argument("--min_length"     ,	default=1     ,	type=_A     ,	help="Min length of the generated answers"							)
	parser.add_argument("--max_length"     ,	default=50     ,	type=_A     ,	help="Max length of the generated answers"							)

	parser.add_argument(
	    "--print_predictions"     ,	action="store_true"     ,	help="If True, prints predictions while evaluating."     ,	)
	parser.add_argument(
	    "--print_docs"     ,	action="store_true"     ,	help="If True, prints docs retried while generating."     ,	)
	snake_case_		    =     parser.parse_args()
	snake_case_		    =     torch.device("cuda" if torch.cuda.is_available() else "cpu"							)
	return args


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	if args.model_type is None:
		snake_case_		    =     infer_model_type(args.model_name_or_path							)
		assert args.model_type is not None
	if args.model_type.startswith("rag"							):
		snake_case_		    =     RagTokenForGeneration if args.model_type == "rag_token" else RagSequenceForGeneration
		snake_case_		    =     args.n_docs
		if args.index_name is not None:
			snake_case_		    =     args.index_name
		if args.index_path is not None:
			snake_case_		    =     args.index_path
	else:
		snake_case_		    =     BartForConditionalGeneration

	snake_case_		    =     (
	    [f.path for f in os.scandir(args.model_name_or_path							) if f.is_dir()]
	    if args.eval_all_checkpoints
	    else [args.model_name_or_path]
	)

	logger.info("Evaluate the following checkpoints: %s"     ,	_A							)

	snake_case_		    =     get_scores if args.eval_mode == "e2e" else get_precision_at_k
	snake_case_		    =     evaluate_batch_eae if args.eval_mode == "e2e" else evaluate_batch_retrieval

	for checkpoint in checkpoints:
		if os.path.exists(args.predictions_path							) and (not args.recalculate):
			logger.info("Calculating metrics based on an existing predictions file: {}".format(args.predictions_path							)							)
			score_fn(_A     ,	args.predictions_path     ,	args.gold_data_path							)
			continue

		logger.info("***** Running evaluation for {} *****".format(_A							)							)
		logger.info("  Batch size = %d"     ,	args.eval_batch_size							)
		logger.info("  Predictions will be stored under {}".format(args.predictions_path							)							)

		if args.model_type.startswith("rag"							):
			snake_case_		    =     RagRetriever.from_pretrained(_A     ,	**_A							)
			snake_case_		    =     model_class.from_pretrained(_A     ,	retriever=_A     ,	**_A							)
			model.retriever.init_retrieval()
		else:
			snake_case_		    =     model_class.from_pretrained(_A     ,	**_A							)
		model.to(args.device							)

		with open(args.evaluation_set     ,	"r"							) as eval_file, open(args.predictions_path     ,	"w"							) as preds_file:
			snake_case_		    =     []
			for line in tqdm(_A							):
				questions.append(line.strip()							)
				if len(_A							) == args.eval_batch_size:
					snake_case_		    =     evaluate_batch_fn(_A     ,	_A     ,	_A							)
					preds_file.write("\n".join(_A							) + "\n"							)
					preds_file.flush()
					snake_case_		    =     []
			if len(_A							) > 0:
				snake_case_		    =     evaluate_batch_fn(_A     ,	_A     ,	_A							)
				preds_file.write("\n".join(_A							)							)
				preds_file.flush()

			score_fn(_A     ,	args.predictions_path     ,	args.gold_data_path							)


if __name__ == "__main__":
		lowercase__				:     Optional[int] 		=						get_args()
		main(args)






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     0






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(GPTaTokenizer, GPTaTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(RobertaTokenizer, RobertaTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	20						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     AutoConfig.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	__lowercase						)
							# Check that tokenizer_type ≠ model_type
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	config=__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with pytest.raises(__lowercase						):
								AutoTokenizer.from_pretrained("./"       ,	tokenizer_type="xxx"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								snake_case_		    =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)

								if isinstance(__lowercase       ,	__lowercase						):
									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,	__lowercase						)
								else:
									self.assertEqual(tokenizer.do_lower_case       ,	__lowercase						)

								self.assertEqual(tokenizer.model_max_length       ,	5_12						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								with self.assertRaisesRegex(
								    __lowercase       ,	"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,	):
									snake_case_		    =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     TOKENIZER_MAPPING.values()
							snake_case_		    =     []

							for slow_tok, fast_tok in tokenizers:
								if slow_tok is not None:
									tokenizer_names.append(slow_tok.__name__						)

								if fast_tok is not None:
									tokenizer_names.append(fast_tok.__name__						)

							for tokenizer_name in tokenizer_names:
								# must find the right class
								tokenizer_class_from_name(__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,	use_fast=__lowercase						)       ,	__lowercase						)
							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     "Hello, world. How are you?"
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)

							snake_case_		    =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)






						@require_tokenizers
						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
							self.assertEqual(type(__lowercase						)       ,	__lowercase						)
							self.assertEqual(tokenizer.model_max_length       ,	5_12						)
							self.assertEqual(tokenizer.vocab_size       ,	3_00_00						)
							self.assertEqual(tokenizer.unk_token       ,	"[UNK]"						)
							self.assertEqual(tokenizer.padding_side       ,	"right"						)
							self.assertEqual(tokenizer.truncation_side       ,	"right"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase       ,	tokenizer.__class__						)
							self.assertEqual(tokenizera.vocab_size       ,	12						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("ctrl"						)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     get_tokenizer_config("bert-base-cased"						)
							snake_case_		    =     config.pop("_commit_hash"       ,	__lowercase						)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__lowercase       ,	{"do_lower_case": False}						)

							# This model does not have a tokenizer_config so we get back an empty dict.
							snake_case_		    =     get_tokenizer_config(__lowercase						)
							self.assertDictEqual(__lowercase       ,	{}						)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     get_tokenizer_config(__lowercase						)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["tokenizer_class"]       ,	"BertTokenizer"						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)

								snake_case_		    =     CustomTokenizer.from_pretrained(__lowercase						)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						@require_tokenizers
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								# Can register in two steps
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, None)						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								del TOKENIZER_MAPPING._extra_content[CustomConfig]
								# Can register in one step
								AutoTokenizer.register(
								    __lowercase       ,	slow_tokenizer_class=__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)

								# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
								# and that model does not have a tokenizer.json
								with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case_		    =     BertTokenizerFast.from_pretrained(__lowercase						)
									bert_tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     CustomTokenizerFast.from_pretrained(__lowercase						)

								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	use_fast=__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)

							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase						)
							self.assertTrue(reloaded_tokenizer.special_attribute_present						)

							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(reloaded_tokenizer.special_attribute_present						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   False







							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   NewTokenizer
													lowerCAmelCase_     		=   False

							try:
								AutoConfig.register("custom"       ,	__lowercase						)
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								# If remote code is not set, the default is to use local
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote code is disabled, we load the local one.
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote is enabled, we load from the Hub
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertTrue(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(tokenizer.special_attribute_present						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(
							    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     AutoTokenizer.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)
