code: 
  """simple docstring"""
  
  
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import pre_tokenizers, processors
  
  from ...tokenization_utils_base import AddedToken, BatchEncoding
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_longformer import LongformerTokenizer
  
  
  _a	 =							logging.get_logger(__name__)
  
  _a	 =							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}
  
  _a	 =							{
      'vocab_file': {
          'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
          'allenai/longformer-large-4096': (
              'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
          ),
          'allenai/longformer-large-4096-finetuned-triviaqa': (
              'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
          ),
          'allenai/longformer-base-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
          ),
          'allenai/longformer-large-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
          ),
      },
      'merges_file': {
          'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
          'allenai/longformer-large-4096': (
              'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
          ),
          'allenai/longformer-large-4096-finetuned-triviaqa': (
              'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
          ),
          'allenai/longformer-base-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
          ),
          'allenai/longformer-large-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
          ),
      },
      'tokenizer_file': {
          'allenai/longformer-base-4096': (
              'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
          ),
          'allenai/longformer-large-4096': (
              'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
          ),
          'allenai/longformer-large-4096-finetuned-triviaqa': (
              'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
          ),
          'allenai/longformer-base-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
          ),
          'allenai/longformer-large-4096-extra.pos.embd.only': (
              'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
          ),
      },
  }
  
  _a	 =							{
      'allenai/longformer-base-4096': 4_096,
      'allenai/longformer-large-4096': 4_096,
      'allenai/longformer-large-4096-finetuned-triviaqa': 4_096,
      'allenai/longformer-base-4096-extra.pos.embd.only': 4_096,
      'allenai/longformer-large-4096-extra.pos.embd.only': 4_096,
  }
  
  
  class A_			(lowercase__   ):
  
  
  
  
    '''simple docstring'''
    SCREAMING_SNAKE_CASE__    :		Optional[int]						  =	VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE__    :		Dict						  =	PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE__    :		Tuple						  =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE__    :		List[Any]						  =	["""input_ids""", """attention_mask"""]
    SCREAMING_SNAKE_CASE__    :		str						  =	LongformerTokenizer
  
  
  
  
  
  
  
    def __init__(	self       ,     lowercase_=None       ,     lowercase_=None       ,     lowercase_=None       ,     lowercase_="replace"       ,     lowercase_="<s>"       ,     lowercase_="</s>"       ,     lowercase_="</s>"       ,     lowercase_="<s>"       ,     lowercase_="<unk>"       ,     lowercase_="<pad>"       ,     lowercase_="<mask>"       ,     lowercase_=False       ,     lowercase_=True       ,     **lowercase_       ,     ):
  
       """simple docstring"""
       super().__init__(
           lowercase_       ,     lowercase_       ,     tokenizer_file=lowercase_       ,     errors=lowercase_       ,     bos_token=lowercase_       ,     eos_token=lowercase_       ,     sep_token=lowercase_       ,     cls_token=lowercase_       ,     unk_token=lowercase_       ,     pad_token=lowercase_       ,     mask_token=lowercase_       ,     add_prefix_space=lowercase_       ,     trim_offsets=lowercase_       ,     **lowercase_       ,     )
  
       UpperCAmelCase_      :      List[str]      =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
       if pre_tok_state.get("add_prefix_space"       ,     lowercase_						) != add_prefix_space:
          UpperCAmelCase_      :      Dict      =					getattr(lowercase_       ,     pre_tok_state.pop("type"						)						)
          UpperCAmelCase_      :      Any      =					add_prefix_space
          UpperCAmelCase_      :      str      =					pre_tok_class(**lowercase_						)
  
       UpperCAmelCase_      :      Dict      =					add_prefix_space
  
       UpperCAmelCase_      :      int      =					"post_processor"
       UpperCAmelCase_      :      Optional[Any]      =					getattr(self.backend_tokenizer       ,     lowercase_       ,     lowercase_						)
       if tokenizer_component_instance:
          UpperCAmelCase_      :      List[Any]      =					json.loads(tokenizer_component_instance.__getstate__()						)
  
          # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
          if "sep" in state:
             UpperCAmelCase_      :      Dict      =					tuple(state["sep"]						)
          if "cls" in state:
             UpperCAmelCase_      :      Tuple      =					tuple(state["cls"]						)
  
          UpperCAmelCase_      :      Tuple      =					False
  
          if state.get("add_prefix_space"       ,     lowercase_						) != add_prefix_space:
             UpperCAmelCase_      :      Dict      =					add_prefix_space
             UpperCAmelCase_      :      List[str]      =					True
  
          if state.get("trim_offsets"       ,     lowercase_						) != trim_offsets:
             UpperCAmelCase_      :      Union[str, Any]      =					trim_offsets
             UpperCAmelCase_      :      List[str]      =					True
  
          if changes_to_apply:
             UpperCAmelCase_      :      List[Any]      =					getattr(lowercase_       ,     state.pop("type"						)						)
             UpperCAmelCase_      :      List[Any]      =					component_class(**lowercase_						)
             setattr(self.backend_tokenizer       ,     lowercase_       ,     lowercase_						)
  
  
  
  
  
  
  
    @property
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       if self._mask_token is None:
          if self.verbose:
             logger.error("Using mask_token, but it is not set yet."						)
          return None
       return str(self._mask_token						)
  
  
  
  
  
  
  
    @mask_token.setter
    def  UpperCamelCase__      (	self       ,     lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Optional[int]      =					AddedToken(lowercase_       ,     lstrip=lowercase_       ,     rstrip=lowercase_						) if isinstance(lowercase_       ,     lowercase_						) else value
       UpperCAmelCase_      :      List[str]      =					value
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     *lowercase_       ,     **lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Union[str, Any]      =					kwargs.get("is_split_into_words"       ,     lowercase_						)
       assert self.add_prefix_space or not is_split_into_words, (
           F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           "to use it with pretokenized inputs."
       )
  
       return super()._batch_encode_plus(*lowercase_       ,     **lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     *lowercase_       ,     **lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Dict      =					kwargs.get("is_split_into_words"       ,     lowercase_						)
  
       assert self.add_prefix_space or not is_split_into_words, (
           F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           "to use it with pretokenized inputs."
       )
  
       return super()._encode_plus(*lowercase_       ,     **lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = None						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Optional[Any]      =					self._tokenizer.model.save(lowercase_       ,     name=lowercase_						)
       return tuple(lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_=None						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Tuple      =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return output
  
       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_ = None						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Optional[int]      =					[self.sep_token_id]
       UpperCAmelCase_      :      Tuple      =					[self.cls_token_id]
  
       if token_ids_a is None:
          return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
  
  
  
style_context: 
  """simple docstring"""
  
  
  
  
  
  from __future__ import annotations
  
  import unittest
  
  from transformers import CTRLConfig, is_tf_available
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
      import tensorflow as tf
  
      from transformers.models.ctrl.modeling_tf_ctrl import (
          TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFCTRLForSequenceClassification,
          TFCTRLLMHeadModel,
          TFCTRLModel,
      )
  
  
  class A_			(lowercase__   ):
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    def __init__(	self       ,     lowercase_       ,     ):
  
       """simple docstring"""
       UpperCAmelCase_      :      List[str]      =					parent
       UpperCAmelCase_      :      List[Any]      =					13
       UpperCAmelCase_      :      Dict      =					7
       UpperCAmelCase_      :      Optional[int]      =					True
       UpperCAmelCase_      :      List[Any]      =					True
       UpperCAmelCase_      :      Optional[int]      =					True
       UpperCAmelCase_      :      int      =					True
       UpperCAmelCase_      :      Any      =					True
       UpperCAmelCase_      :      Any      =					99
       UpperCAmelCase_      :      Dict      =					32
       UpperCAmelCase_      :      Dict      =					2
       UpperCAmelCase_      :      str      =					4
       UpperCAmelCase_      :      int      =					37
       UpperCAmelCase_      :      Union[str, Any]      =					"gelu"
       UpperCAmelCase_      :      List[str]      =					0.1
       UpperCAmelCase_      :      List[str]      =					0.1
       UpperCAmelCase_      :      int      =					512
       UpperCAmelCase_      :      str      =					16
       UpperCAmelCase_      :      Union[str, Any]      =					2
       UpperCAmelCase_      :      Dict      =					0.02
       UpperCAmelCase_      :      Tuple      =					3
       UpperCAmelCase_      :      Tuple      =					4
       UpperCAmelCase_      :      int      =					None
       UpperCAmelCase_      :      Dict      =					self.vocab_size - 1
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Dict      =					ids_tensor([self.batch_size, self.seq_length]       ,     self.vocab_size						)
  
       UpperCAmelCase_      :      Optional[int]      =					None
       if self.use_input_mask:
          UpperCAmelCase_      :      Optional[Any]      =					random_attention_mask([self.batch_size, self.seq_length]						)
  
       UpperCAmelCase_      :      List[Any]      =					None
       if self.use_token_type_ids:
          UpperCAmelCase_      :      Optional[int]      =					ids_tensor([self.batch_size, self.seq_length]       ,     self.type_vocab_size						)
  
       UpperCAmelCase_      :      str      =					None
       if self.use_mc_token_ids:
          UpperCAmelCase_      :      Any      =					ids_tensor([self.batch_size, self.num_choices]       ,     self.seq_length						)
  
       UpperCAmelCase_      :      Union[str, Any]      =					None
       UpperCAmelCase_      :      str      =					None
       UpperCAmelCase_      :      str      =					None
       if self.use_labels:
          UpperCAmelCase_      :      List[Any]      =					ids_tensor([self.batch_size]       ,     self.type_sequence_label_size						)
          UpperCAmelCase_      :      str      =					ids_tensor([self.batch_size, self.seq_length]       ,     self.num_labels						)
          UpperCAmelCase_      :      Optional[Any]      =					ids_tensor([self.batch_size]       ,     self.num_choices						)
  
       UpperCAmelCase_      :      str      =					CTRLConfig(
           vocab_size=self.vocab_size       ,     n_embd=self.hidden_size       ,     n_layer=self.num_hidden_layers       ,     n_head=self.num_attention_heads       ,     n_positions=self.max_position_embeddings       ,     pad_token_id=self.pad_token_id       ,     )
  
       UpperCAmelCase_      :      Tuple      =					ids_tensor([self.num_hidden_layers, self.num_attention_heads]       ,     2						)
  
       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     *lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      int      =					TFCTRLModel(config=lowercase_						)
       UpperCAmelCase_      :      Optional[Any]      =					{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase_      :      Union[str, Any]      =					model(lowercase_						)
  
       UpperCAmelCase_      :      Union[str, Any]      =					[input_ids, None, input_mask]  # None is the input for 'past'
       UpperCAmelCase_      :      str      =					model(lowercase_						)
  
       UpperCAmelCase_      :      str      =					model(lowercase_						)
  
       self.parent.assertEqual(result.last_hidden_state.shape       ,     (self.batch_size, self.seq_length, self.hidden_size)						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     *lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Optional[int]      =					TFCTRLLMHeadModel(config=lowercase_						)
       UpperCAmelCase_      :      List[Any]      =					{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       UpperCAmelCase_      :      List[str]      =					model(lowercase_						)
       self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.seq_length, self.vocab_size)						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     *lowercase_						):
  
       """simple docstring"""
       UpperCAmelCase_      :      List[str]      =					self.num_labels
       UpperCAmelCase_      :      str      =					ids_tensor([self.batch_size]       ,     self.type_sequence_label_size						)
       UpperCAmelCase_      :      Union[str, Any]      =					{
           "input_ids": input_ids,
           "token_type_ids": token_type_ids,
           "labels": sequence_labels,
       }
       UpperCAmelCase_      :      Union[str, Any]      =					TFCTRLForSequenceClassification(lowercase_						)
       UpperCAmelCase_      :      Tuple      =					model(lowercase_						)
       self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_labels)						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Dict      =					self.prepare_config_and_inputs()
  
       (
           (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     (
           UpperCAmelCase_
       )  ,     
       )      :      Tuple      =					config_and_inputs
  
       UpperCAmelCase_      :      int      =					{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict
  
  
  
  
  @require_tf
  class A_			(lowercase__       ,lowercase__       ,unittest.TestCase   ):
  
  
  
  
    '''simple docstring'''
    SCREAMING_SNAKE_CASE__    :		Dict						  =	(TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
    SCREAMING_SNAKE_CASE__    :		str						  =	(TFCTRLLMHeadModel,) if is_tf_available() else ()
    SCREAMING_SNAKE_CASE__    :		Any						  =	(
        {
            """feature-extraction""": TFCTRLModel,
            """text-classification""": TFCTRLForSequenceClassification,
            """text-generation""": TFCTRLLMHeadModel,
            """zero-shot""": TFCTRLForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    SCREAMING_SNAKE_CASE__    :		Dict						  =	False
    SCREAMING_SNAKE_CASE__    :		List[Any]						  =	False
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_       ,     lowercase_						):
  
       """simple docstring"""
       if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
          # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
          # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
          # config could not be created.
          return True
  
       return False
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      str      =					TFCTRLModelTester(self						)
       UpperCAmelCase_      :      Dict      =					ConfigTester(self       ,     config_class=lowercase_       ,     n_embd=37						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       self.config_tester.run_common_tests()
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      str      =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_ctrl_model(*lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Union[str, Any]      =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_ctrl_lm_head(*lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Optional[Any]      =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_ctrl_for_sequence_classification(*lowercase_						)
  
  
  
  
  
  
  
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_  ,     UpperCAmelCase_      :      List[Any]      =					self.model_tester.prepare_config_and_inputs_for_common()
       UpperCAmelCase_      :      Tuple      =					[TFCTRLLMHeadModel]
       UpperCAmelCase_      :      Dict      =					[TFCTRLForSequenceClassification]
  
       for model_class in self.all_model_classes:
          UpperCAmelCase_      :      Optional[int]      =					model_class(lowercase_						)
          model.build()  # may be needed for the get_bias() call below
          assert isinstance(model.get_input_embeddings()       ,     tf.keras.layers.Layer						)
  
          if model_class in list_lm_models:
             UpperCAmelCase_      :      List[str]      =					model.get_output_embeddings()
             assert isinstance(lowercase_       ,     tf.keras.layers.Layer						)
             UpperCAmelCase_      :      List[Any]      =					model.get_bias()
             assert isinstance(lowercase_       ,     lowercase_						)
             for k, v in name.items():
                assert isinstance(lowercase_       ,     tf.Variable						)
          elif model_class in list_other_models_with_output_ebd:
             UpperCAmelCase_      :      int      =					model.get_output_embeddings()
             assert isinstance(lowercase_       ,     tf.keras.layers.Layer						)
             UpperCAmelCase_      :      Any      =					model.get_bias()
             assert name is None
          else:
             UpperCAmelCase_      :      Dict      =					model.get_output_embeddings()
             assert x is None
             UpperCAmelCase_      :      Tuple      =					model.get_bias()
             assert name is None
  
  
  
  
  
  
  
    @slow
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          UpperCAmelCase_      :      List[Any]      =					TFCTRLModel.from_pretrained(lowercase_						)
          self.assertIsNotNone(lowercase_						)
  
  
  
  
  @require_tf
  class A_			(unittest.TestCase   ):
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    @slow
    def  UpperCamelCase__      (	self						):
  
       """simple docstring"""
       UpperCAmelCase_      :      Union[str, Any]      =					TFCTRLLMHeadModel.from_pretrained("ctrl"						)
       UpperCAmelCase_      :      Union[str, Any]      =					tf.convert_to_tensor([[1_1859, 0, 1611, 8]]       ,     dtype=tf.intaa						)  # Legal the president is
       UpperCAmelCase_      :      int      =					[
           1_1859,
           0,
           1611,
           8,
           5,
           150,
           2_6449,
           2,
           19,
           348,
           469,
           3,
           2595,
           48,
           2_0740,
           24_6533,
           24_6533,
           19,
           30,
           5,
       ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a
  
       UpperCAmelCase_      :      List[str]      =					model.generate(lowercase_       ,     do_sample=lowercase_						)
       self.assertListEqual(output_ids[0].numpy().tolist()       ,     lowercase_						)
  
  
  