import argparse
import random

import joblib
import numpy as np
import torch
from igf.igf import (
    SecondaryLearner,
    collect_objective_set,
    compute_perplexity,
    generate_datasets,
    load_gpta,
    recopy_gpta,
    set_seed,
    train_secondary_learner,
)
from torch.utils.data import DataLoader, RandomSampler

from transformers import GPTaLMHeadModel



def lowerCamelCase  (   SCREAMING_SNAKE_CASE=32  ,			SCREAMING_SNAKE_CASE=10  ,			SCREAMING_SNAKE_CASE=100  ,			SCREAMING_SNAKE_CASE=1_026  ,			SCREAMING_SNAKE_CASE=True  ,			SCREAMING_SNAKE_CASE="data/tokenized_stories_train_wikitext103.jbl"  ,			SCREAMING_SNAKE_CASE="igf_context_pairs.jbl"  ,			):
    '''simple docstring'''







    set_seed(3	)
    # generate train_data and objective_set
    __UpperCamelCase  ,       __UpperCamelCase     :Optional[Any]			  =		generate_datasets(
        SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			number=SCREAMING_SNAKE_CASE  ,			min_len=1_026  ,			trim=SCREAMING_SNAKE_CASE	)
    # keeps model same across runs
    set_seed(4	)
    # model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights
    # can we train on GPU?
    __UpperCamelCase     :List[Any]			  =		torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''	)

    # load pretrained model
    __UpperCamelCase     :str			  =		load_gpta('''gpt2'''	).to(SCREAMING_SNAKE_CASE	)
    print('''computing perplexity on objective set'''	)
    __UpperCamelCase     :List[str]			  =		compute_perplexity(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	).item()
    print('''perplexity on objective set:'''  ,			SCREAMING_SNAKE_CASE	)

    # collect igf pairs and save to file demo.jbl
    collect_objective_set(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)

    # clean up, delete model and data we don't need anymore
    del model, train_data, objective_set
    torch.cuda.empty_cache()



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=15  ,			SCREAMING_SNAKE_CASE=128  ,			SCREAMING_SNAKE_CASE=100  ,			SCREAMING_SNAKE_CASE="igf_model.pt"  ,			):
    '''simple docstring'''







    set_seed(42	)

    # Load pre-trained model
    __UpperCamelCase     :str			  =		GPTaLMHeadModel.from_pretrained('''gpt2'''	)

    # Initialize secondary learner to use embedding weights of model
    __UpperCamelCase     :List[str]			  =		SecondaryLearner(SCREAMING_SNAKE_CASE	)

    # Train secondary learner
    __UpperCamelCase     :Tuple			  =		train_secondary_learner(
        SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			max_epochs=SCREAMING_SNAKE_CASE  ,			batch_size=SCREAMING_SNAKE_CASE  ,			eval_freq=100  ,			igf_model_path=SCREAMING_SNAKE_CASE  ,			)

    del model, secondary_learner_train_data
    torch.cuda.empty_cache()

    return secondary_learner



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=32  ,			SCREAMING_SNAKE_CASE=1_000  ,			SCREAMING_SNAKE_CASE=16  ,			SCREAMING_SNAKE_CASE=1.0  ,			SCREAMING_SNAKE_CASE=recopy_gpta  ,			SCREAMING_SNAKE_CASE=None  ,			SCREAMING_SNAKE_CASE=10  ,			SCREAMING_SNAKE_CASE="gpt2_finetuned.pt"  ,			):
    '''simple docstring'''







    __UpperCamelCase     :List[Any]			  =		torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''	)
    __UpperCamelCase     :Tuple			  =		RandomSampler(SCREAMING_SNAKE_CASE	)
    __UpperCamelCase     :Union[str, Any]			  =		DataLoader(SCREAMING_SNAKE_CASE  ,			sampler=SCREAMING_SNAKE_CASE	)

    __UpperCamelCase     :List[Any]			  =		max_steps // (len(SCREAMING_SNAKE_CASE	)) + 1
    __UpperCamelCase     :Optional[int]			  =		0
    __UpperCamelCase     :int			  =		torch.zeros((1, context_len)  ,			dtype=torch.long  ,			device=SCREAMING_SNAKE_CASE	)
    __UpperCamelCase  ,       __UpperCamelCase  ,       __UpperCamelCase     :List[str]			  =		recopy_model(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)

    model.train()
    if secondary_learner is not None:
        secondary_learner.to(SCREAMING_SNAKE_CASE	)
        secondary_learner.eval()
    __UpperCamelCase     :List[str]			  =		[]
    __UpperCamelCase     :str			  =		0

    __UpperCamelCase     :int			  =		[]
    __UpperCamelCase     :int			  =		[]

    # Compute the performance of the transformer model at the beginning
    __UpperCamelCase     :List[str]			  =		compute_perplexity(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)
    test_perps.append(SCREAMING_SNAKE_CASE	)
    print('''Test perplexity, step'''  ,			SCREAMING_SNAKE_CASE  ,			''':'''  ,			SCREAMING_SNAKE_CASE	)
    for epoch in range(int(SCREAMING_SNAKE_CASE	)	):
        for step, example in enumerate(SCREAMING_SNAKE_CASE	):
            torch.cuda.empty_cache()
            __UpperCamelCase     :Optional[Any]			  =		random.randint(0  ,			example.size(2	) - context_len - 1	)
            __UpperCamelCase     :Tuple			  =		example[0, 0, start : start + context_len]
            lm_optimizer.zero_grad()
            __UpperCamelCase     :List[str]			  =		model(SCREAMING_SNAKE_CASE  ,			labels=SCREAMING_SNAKE_CASE	)
            __UpperCamelCase     :Any			  =		True

            if secondary_learner is not None:
                __UpperCamelCase     :List[Any]			  =		secondary_learner.forward(
                    torch.tensor(SCREAMING_SNAKE_CASE  ,			dtype=torch.long  ,			device=SCREAMING_SNAKE_CASE	).unsqueeze(0	)	)[0].item()
                observed_qs.append(float(SCREAMING_SNAKE_CASE	)	)

                # Here we implement the simple non-constant threshold for the predicted IG(X) value
                # We will decay the selectivity of our secondary learner filter from
                # 1 standard deviation above average to 1 below average after 10 batches.

                if global_step == 10:
                    __UpperCamelCase     :List[Any]			  =		-1
                if predicted_q < threshold:
                    __UpperCamelCase     :List[str]			  =		False

            # If we passed the filter, add the context to the batch!
            if do_backprop:
                contexts.append(np.array(context.cpu()	)	)
                __UpperCamelCase     :int			  =		outputs[0]
                lm_loss.backward()
                examples += 1

            del outputs

            # Once the batch is filled with enough contexts, backprop on the batch.
            if examples == batch_size:
                torch.cuda.empty_cache()
                __UpperCamelCase     :Any			  =		0
                # Do LM backprop
                torch.nn.utils.clip_grad_norm_(model.parameters()  ,			3.0	)
                lm_optimizer.step()
                lm_scheduler.step()  # Update learning rate schedule
                global_step += 1
                # Compute the performance of the transformer model at this batch
                if global_step % eval_interval == 0:
                    __UpperCamelCase     :Tuple			  =		compute_perplexity(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)
                    test_perps.append(SCREAMING_SNAKE_CASE	)

                    print('''Test perplexity, step'''  ,			SCREAMING_SNAKE_CASE  ,			''':'''  ,			SCREAMING_SNAKE_CASE	)
            # Break out of the loop after 60 batches
            if max_steps > 0 and global_step > 60:
                break
        if max_steps > 0 and global_step > 60:
            break

    # save finetuned transformer model
    torch.save(model.state_dict()  ,			SCREAMING_SNAKE_CASE	)
    torch.cuda.empty_cache()
    # Do some cleaning up so we can reinitialize for the next run of this function
    del lm_optimizer
    del lm_scheduler
    return model



def lowerCamelCase  (   ):
    '''simple docstring'''







    __UpperCamelCase     :List[str]			  =		argparse.ArgumentParser(description='''Fine-tune a transformer model with IGF on a language modeling task'''	)

    # Required parameters
    parser.add_argument(
        '''--data_dir'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			required=SCREAMING_SNAKE_CASE  ,			help='''The input data dir. Should contain data files for WikiText.'''  ,			)
    parser.add_argument(
        '''--model_name_or_path'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			required=SCREAMING_SNAKE_CASE  ,			help='''Path to pretrained model or model identifier from huggingface.co/models'''  ,			)
    parser.add_argument(
        '''--data_file'''  ,			type=SCREAMING_SNAKE_CASE  ,			default=SCREAMING_SNAKE_CASE  ,			help=(
            '''A jbl file containing tokenized data which can be split as objective dataset, '''
            '''train_dataset and test_dataset.'''
        )  ,			)

    parser.add_argument(
        '''--igf_data_file'''  ,			type=SCREAMING_SNAKE_CASE  ,			default=SCREAMING_SNAKE_CASE  ,			help='''A jbl file containing the context and information gain pairs to train secondary learner.'''  ,			)

    parser.add_argument(
        '''--output_dir'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			required=SCREAMING_SNAKE_CASE  ,			help='''The output directory where the final fine-tuned model is stored.'''  ,			)

    parser.add_argument(
        '''--tokenizer_name'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			help='''Pretrained tokenizer name or path if not the same as model_name'''  ,			)
    parser.add_argument('''--seed'''  ,			type=SCREAMING_SNAKE_CASE  ,			default=SCREAMING_SNAKE_CASE  ,			help='''A seed for reproducible training.'''	)

    parser.add_argument(
        '''--context_len'''  ,			default=32  ,			type=SCREAMING_SNAKE_CASE  ,			help=(
            '''The maximum total input sequence length after tokenization. Sequences longer '''
            '''than this will be truncated, sequences shorter will be padded.'''
        )  ,			)

    parser.add_argument(
        '''--size_objective_set'''  ,			default=100  ,			type=SCREAMING_SNAKE_CASE  ,			help='''number of articles that are long enough to be used as our objective set'''  ,			)
    parser.add_argument(
        '''--eval_freq'''  ,			default=100  ,			type=SCREAMING_SNAKE_CASE  ,			help='''secondary model evaluation is triggered at eval_freq'''	)

    parser.add_argument('''--max_steps'''  ,			default=1_000  ,			type=SCREAMING_SNAKE_CASE  ,			help='''To calculate training epochs'''	)

    parser.add_argument(
        '''--secondary_learner_batch_size'''  ,			default=128  ,			type=SCREAMING_SNAKE_CASE  ,			help='''batch size of training data for secondary learner'''  ,			)

    parser.add_argument(
        '''--batch_size'''  ,			default=16  ,			type=SCREAMING_SNAKE_CASE  ,			help='''batch size of training data of language model(gpt2) '''	)

    parser.add_argument(
        '''--eval_interval'''  ,			default=10  ,			type=SCREAMING_SNAKE_CASE  ,			help=(
            '''decay the selectivity of our secondary learner filter from'''
            '''1 standard deviation above average to 1 below average after 10 batches'''
        )  ,			)

    parser.add_argument(
        '''--number'''  ,			default=100  ,			type=SCREAMING_SNAKE_CASE  ,			help='''The number of examples split to be used as objective_set/test_data'''	)

    parser.add_argument(
        '''--min_len'''  ,			default=1_026  ,			type=SCREAMING_SNAKE_CASE  ,			help='''The minimum length of the article to be used as objective set'''	)

    parser.add_argument(
        '''--secondary_learner_max_epochs'''  ,			default=15  ,			type=SCREAMING_SNAKE_CASE  ,			help='''number of epochs to train secondary learner'''	)

    parser.add_argument('''--trim'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			help='''truncate the example if it exceeds context length'''	)

    parser.add_argument(
        '''--threshold'''  ,			default=1.0  ,			type=SCREAMING_SNAKE_CASE  ,			help=(
            '''The threshold value used by secondary learner to filter the train_data and allow only'''
            ''' informative data as input to the model'''
        )  ,			)

    parser.add_argument('''--finetuned_model_name'''  ,			default='''gpt2_finetuned.pt'''  ,			type=SCREAMING_SNAKE_CASE  ,			help='''finetuned_model_name'''	)

    parser.add_argument(
        '''--recopy_model'''  ,			default=SCREAMING_SNAKE_CASE  ,			type=SCREAMING_SNAKE_CASE  ,			help='''Reset the model to the original pretrained GPT-2 weights after each iteration'''  ,			)

    # function calls
    # Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner
    generate_n_pairs(
        context_len=32  ,			max_steps=10  ,			size_objective_set=100  ,			min_len=1_026  ,			trim=SCREAMING_SNAKE_CASE  ,			data_file='''data/tokenized_stories_train_wikitext103.jbl'''  ,			igf_data_file='''igf_context_pairs.jbl'''  ,			)

    # Load train data for secondary learner
    __UpperCamelCase     :Optional[Any]			  =		joblib.load('''data/IGF_values.jbl'''	)

    # Train secondary learner
    __UpperCamelCase     :str			  =		training_secondary_learner(
        SCREAMING_SNAKE_CASE  ,			secondary_learner_max_epochs=15  ,			secondary_learner_batch_size=128  ,			eval_freq=100  ,			igf_model_path='''igf_model.pt'''  ,			)

    # load pretrained gpt2 model
    __UpperCamelCase     :Union[str, Any]			  =		GPTaLMHeadModel.from_pretrained('''gpt2'''	)
    set_seed(42	)

    # Generate train and test data to train and evaluate gpt2 model
    __UpperCamelCase  ,       __UpperCamelCase     :Dict			  =		generate_datasets(
        context_len=32  ,			file='''data/tokenized_stories_train_wikitext103.jbl'''  ,			number=100  ,			min_len=1_026  ,			trim=SCREAMING_SNAKE_CASE	)

    # fine-tuning of the gpt2 model using igf (Information Gain Filtration)
    finetune(
        SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			context_len=32  ,			max_steps=1_000  ,			batch_size=16  ,			threshold=1.0  ,			recopy_model=SCREAMING_SNAKE_CASE  ,			secondary_learner=SCREAMING_SNAKE_CASE  ,			eval_interval=10  ,			finetuned_model_name='''gpt2_finetuned.pt'''  ,			)


if __name__ == "__main__":
    main()











import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     0






						@slow
						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)

							for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)
								self.assertIsInstance(__lowercase       ,	(GPTaTokenizer, GPTaTokenizerFast)						)
								self.assertGreater(len(__lowercase						)       ,	0						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(RobertaTokenizer, RobertaTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	20						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     AutoConfig.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	__lowercase						)
							# Check that tokenizer_type ≠ model_type
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	config=__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							self.assertEqual(tokenizer.vocab_size       ,	12						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"       ,	use_fast=__lowercase						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.txt"       ,	os.path.join(__lowercase       ,	"vocab.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="bert"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								shutil.copy("./tests/fixtures/vocab.json"       ,	os.path.join(__lowercase       ,	"vocab.json"						)						)
								shutil.copy("./tests/fixtures/merges.txt"       ,	os.path.join(__lowercase       ,	"merges.txt"						)						)

								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	tokenizer_type="gpt2"						)
								self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with pytest.raises(__lowercase						):
								AutoTokenizer.from_pretrained("./"       ,	tokenizer_type="xxx"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								snake_case_		    =     tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"						)
								self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)

								if isinstance(__lowercase       ,	__lowercase						):
									self.assertEqual(tokenizer.basic_tokenizer.do_lower_case       ,	__lowercase						)
								else:
									self.assertEqual(tokenizer.do_lower_case       ,	__lowercase						)

								self.assertEqual(tokenizer.model_max_length       ,	5_12						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
								with self.assertRaisesRegex(
								    __lowercase       ,	"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"       ,	):
									snake_case_		    =     tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     TOKENIZER_MAPPING.values()
							snake_case_		    =     []

							for slow_tok, fast_tok in tokenizers:
								if slow_tok is not None:
									tokenizer_names.append(slow_tok.__name__						)

								if fast_tok is not None:
									tokenizer_names.append(fast_tok.__name__						)

							for tokenizer_name in tokenizer_names:
								# must find the right class
								tokenizer_class_from_name(__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"       ,	use_fast=__lowercase						)       ,	__lowercase						)
							self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"						)       ,	__lowercase						)






						@require_tokenizers
						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("distilbert-base-uncased"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     "Hello, world. How are you?"
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)

							snake_case_		    =     AutoTokenizer.from_pretrained("microsoft/mpnet-base"       ,	do_lower_case=__lowercase						)
							snake_case_		    =     tokenizer.tokenize(__lowercase						)
							self.assertEqual("[UNK]"       ,	tokens[0]						)






						@require_tokenizers
						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"						)
							self.assertEqual(type(__lowercase						)       ,	__lowercase						)
							self.assertEqual(tokenizer.model_max_length       ,	5_12						)
							self.assertEqual(tokenizer.vocab_size       ,	3_00_00						)
							self.assertEqual(tokenizer.unk_token       ,	"[UNK]"						)
							self.assertEqual(tokenizer.padding_side       ,	"right"						)
							self.assertEqual(tokenizer.truncation_side       ,	"right"						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							self.assertIsInstance(__lowercase       ,	(BertTokenizer, BertTokenizerFast)						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)

							self.assertIsInstance(__lowercase       ,	tokenizer.__class__						)
							self.assertEqual(tokenizera.vocab_size       ,	12						)






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("ctrl"						)
							# There is no fast CTRL so this always gives us a slow tokenizer.
							self.assertIsInstance(__lowercase       ,	__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     get_tokenizer_config("bert-base-cased"						)
							snake_case_		    =     config.pop("_commit_hash"       ,	__lowercase						)
							# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
							self.assertEqual(__lowercase       ,	{"do_lower_case": False}						)

							# This model does not have a tokenizer_config so we get back an empty dict.
							snake_case_		    =     get_tokenizer_config(__lowercase						)
							self.assertDictEqual(__lowercase       ,	{}						)

							# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
							snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     get_tokenizer_config(__lowercase						)

							# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
							self.assertEqual(config["tokenizer_class"]       ,	"BertTokenizer"						)






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)

								snake_case_		    =     CustomTokenizer.from_pretrained(__lowercase						)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						@require_tokenizers
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							try:
								AutoConfig.register("custom"       ,	__lowercase						)

								# Can register in two steps
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, None)						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								del TOKENIZER_MAPPING._extra_content[CustomConfig]
								# Can register in one step
								AutoTokenizer.register(
								    __lowercase       ,	slow_tokenizer_class=__lowercase       ,	fast_tokenizer_class=__lowercase						)
								self.assertEqual(TOKENIZER_MAPPING[CustomConfig]       ,	(CustomTokenizer, CustomTokenizerFast)						)

								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__lowercase						):
									AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)

								# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
								# and that model does not have a tokenizer.json
								with tempfile.TemporaryDirectory() as tmp_dir:
									snake_case_		    =     BertTokenizerFast.from_pretrained(__lowercase						)
									bert_tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     CustomTokenizerFast.from_pretrained(__lowercase						)

								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	use_fast=__lowercase						)
									self.assertIsInstance(__lowercase       ,	__lowercase						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
							# If remote code is disabled, we can't load this config.
							with self.assertRaises(__lowercase						):
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)

							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							# Test tokenizer can be reloaded.
							with tempfile.TemporaryDirectory() as tmp_dir:
								tokenizer.save_pretrained(__lowercase						)
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase						)
							self.assertTrue(reloaded_tokenizer.special_attribute_present						)

							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__lowercase						)
									snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(reloaded_tokenizer.special_attribute_present						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertEqual(reloaded_tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						@require_tokenizers
						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   False







							class 		UpperCAmelCase   (				UpperCAmelCase__			):







													'''simple docstring'''





													lowerCAmelCase_     		=   NewTokenizer
													lowerCAmelCase_     		=   False

							try:
								AutoConfig.register("custom"       ,	__lowercase						)
								AutoTokenizer.register(__lowercase       ,	slow_tokenizer_class=__lowercase						)
								AutoTokenizer.register(__lowercase       ,	fast_tokenizer_class=__lowercase						)
								# If remote code is not set, the default is to use local
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote code is disabled, we load the local one.
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertFalse(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertFalse(tokenizer.special_attribute_present						)

								# If remote is enabled, we load from the Hub
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)
								self.assertTrue(tokenizer.special_attribute_present						)
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
								self.assertTrue(tokenizer.special_attribute_present						)

							finally:
								if "custom" in CONFIG_MAPPING._extra_content:
									del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
									del TOKENIZER_MAPPING._extra_content[CustomConfig]






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained(
							    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase						)
							self.assertTrue(tokenizer.special_attribute_present						)
							if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizerFast"						)

								# Test we can also load the slow version
								snake_case_		    =     AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"       ,	trust_remote_code=__lowercase       ,	use_fast=__lowercase						)
								self.assertTrue(tokenizer.special_attribute_present						)
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)
							else:
								self.assertEqual(tokenizer.__class__.__name__       ,	"NewTokenizer"						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	"bert-base is not a local folder and is not a valid model identifier"						):
								snake_case_		    =     AutoTokenizer.from_pretrained("bert-base"						)






						def        snake_case__						(						self							:     Union[str, Any]						):






							"""simple docstring"""






							with self.assertRaisesRegex(
							    __lowercase       ,	r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"						):
								snake_case_		    =     AutoTokenizer.from_pretrained(__lowercase       ,	revision="aaaaaa"						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
							with RequestCounter() as counter:
								snake_case_		    =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
								self.assertEqual(counter.get_request_count       ,	0						)
								self.assertEqual(counter.head_request_count       ,	1						)
								self.assertEqual(counter.other_request_count       ,	0						)
