code: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  a     :			Optional[int]					    =						logging.get_logger(__name__)
  
  a     :			Optional[Any]					    =						{
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  a     :			Any					    =						{
      """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
      """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
  }
  
  a     :			Any					    =						{
      """openai-gpt""": 5_1_2,
  }
  def 		snake_case__  (		lowercase				):
  
     '''simple docstring'''
  
  
  
  
  
     lowerCAmelCase_:       List[str]         =  text.strip()
     if not text:
        return []
     lowerCAmelCase_:       List[Any]         =  text.split()
     return tokens
  
  class 				_lowercase						(    UpperCAmelCase__ ):
  
  
  
  
  
  
      '''simple docstring'''
  
  
  
  
      def __init__(						self					,	lowerCamelCase__=True					,	lowerCamelCase__=None					,	lowerCamelCase__=True					,	lowerCamelCase__=None					,	lowerCamelCase__=True					,	):
  
  
  
  
  
  
         if never_split is None:
            lowerCAmelCase_:       Tuple         =  []
         lowerCAmelCase_:       str         =  do_lower_case
         lowerCAmelCase_:       Optional[Any]         =  set(lowerCamelCase__   )
         lowerCAmelCase_:       Dict         =  tokenize_chinese_chars
         lowerCAmelCase_:       str         =  strip_accents
         lowerCAmelCase_:       int         =  do_split_on_punc
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__=None   ):
         lowerCAmelCase_:       int         =  self.never_split.union(set(lowerCamelCase__   )   ) if never_split else self.never_split
         lowerCAmelCase_:       Tuple         =  self._clean_text(lowerCamelCase__   )
  
         # This was added on November 1st, 2018 for the multilingual and Chinese
         # models. This is also applied to the English models now, but it doesn't
         # matter since the English models were not trained on any Chinese data
         # and generally don't have any Chinese data in them (there are Chinese
         # characters in the vocabulary because Wikipedia does have some Chinese
         # words in the English Wikipedia.).
         if self.tokenize_chinese_chars:
            lowerCAmelCase_:       int         =  self._tokenize_chinese_chars(lowerCamelCase__   )
         # prevents treating the same character with different unicode codepoints as different characters
         lowerCAmelCase_:       List[Any]         =  unicodedata.normalize("NFC"					,	lowerCamelCase__   )
         lowerCAmelCase_:       Tuple         =  whitespace_tokenize(lowerCamelCase__   )
         lowerCAmelCase_:       Union[str, Any]         =  []
         for token in orig_tokens:
            if token not in never_split:
               if self.do_lower_case:
                  lowerCAmelCase_:       Optional[int]         =  token.lower()
                  if self.strip_accents is not False:
                     lowerCAmelCase_:       Optional[int]         =  self._run_strip_accents(lowerCamelCase__   )
               elif self.strip_accents:
                  lowerCAmelCase_:       List[str]         =  self._run_strip_accents(lowerCamelCase__   )
            split_tokens.extend(self._run_split_on_punc(lowerCamelCase__					,	lowerCamelCase__   )   )
  
         lowerCAmelCase_:       Optional[Any]         =  whitespace_tokenize(" ".join(lowerCamelCase__   )   )
         return output_tokens
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Tuple         =  unicodedata.normalize("NFD"					,	lowerCamelCase__   )
         lowerCAmelCase_:       List[Any]         =  []
         for char in text:
            lowerCAmelCase_:       Tuple         =  unicodedata.category(lowerCamelCase__   )
            if cat == "Mn":
               continue
            output.append(lowerCamelCase__   )
         return "".join(lowerCamelCase__   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__=None   ):
  
  
  
  
  
  
         if not self.do_split_on_punc or (never_split is not None and text in never_split):
            return [text]
         lowerCAmelCase_:       Union[str, Any]         =  list(lowerCamelCase__   )
         lowerCAmelCase_:       Optional[Any]         =  0
         lowerCAmelCase_:       Optional[int]         =  True
         lowerCAmelCase_:       Any         =  []
         while i < len(lowerCamelCase__   ):
            lowerCAmelCase_:       str         =  chars[i]
            if _is_punctuation(lowerCamelCase__   ):
               output.append([char]   )
               lowerCAmelCase_:       Any         =  True
            else:
               if start_new_word:
                  output.append([]   )
               lowerCAmelCase_:       Dict         =  False
               output[-1].append(lowerCamelCase__   )
            i += 1
  
         return ["".join(lowerCamelCase__   ) for x in output]
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Optional[int]         =  []
         for char in text:
            lowerCAmelCase_:       Tuple         =  ord(lowerCamelCase__   )
            if self._is_chinese_char(lowerCamelCase__   ):
               output.append(" "   )
               output.append(lowerCamelCase__   )
               output.append(" "   )
            else:
               output.append(lowerCamelCase__   )
         return "".join(lowerCamelCase__   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
  
  
  
  
  
  
         if (
             (cp >= 0x4_E00 and cp <= 0x9_FFF)
             or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
             or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
             or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
             or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
             or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
             or (cp >= 0xF_900 and cp <= 0xF_AFF)
             or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
         ):  #
            return True
  
         return False
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Tuple         =  []
         for char in text:
            lowerCAmelCase_:       Union[str, Any]         =  ord(lowerCamelCase__   )
            if cp == 0 or cp == 0xF_FFD or _is_control(lowerCamelCase__   ):
               continue
            if _is_whitespace(lowerCamelCase__   ):
               output.append(" "   )
            else:
               output.append(lowerCamelCase__   )
         return "".join(lowerCamelCase__   )
  
  
  
  
  
  def 		snake_case__  (		lowercase				):
  
     '''simple docstring'''
  
  
  
  
  
     lowerCAmelCase_:       int         =  set()
     lowerCAmelCase_:       Dict         =  word[0]
     for char in word[1:]:
        pairs.add((prev_char, char)				)
        lowerCAmelCase_:       List[str]         =  char
     return pairs
  def 		snake_case__  (		lowercase				):
  
     '''simple docstring'''
  
  
  
  
  
     lowerCAmelCase_:       List[Any]         =  text.replace("—"       ,							"-"				)
     lowerCAmelCase_:       Optional[Any]         =  text.replace("–"       ,							"-"				)
     lowerCAmelCase_:       List[Any]         =  text.replace("―"       ,							"-"				)
     lowerCAmelCase_:       Any         =  text.replace("…"       ,							"..."				)
     lowerCAmelCase_:       str         =  text.replace("´"       ,							"'"				)
     lowerCAmelCase_:       str         =  re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"       ,							r" \1 "       ,							lowercase				)
     lowerCAmelCase_:       Union[str, Any]         =  re.sub(r"\s*\n\s*"       ,							" \n "       ,							lowercase				)
     lowerCAmelCase_:       Any         =  re.sub(r"[^\S\n]+"       ,							" "       ,							lowercase				)
     return text.strip()
  
  
  
  
  
  
  
  class 				_lowercase						(    UpperCAmelCase__ ):
  
  
  
  
  
  
      '''simple docstring'''
      SCREAMING_SNAKE_CASE:							List[str]        =						VOCAB_FILES_NAMES
      SCREAMING_SNAKE_CASE:							int        =						PRETRAINED_VOCAB_FILES_MAP
      SCREAMING_SNAKE_CASE:							int        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      SCREAMING_SNAKE_CASE:							Union[str, Any]        =						["input_ids", "attention_mask"]
  
  
  
  
      def __init__(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__="<unk>"					,	**lowerCamelCase__   ):
         super().__init__(unk_token=lowerCamelCase__					,	**lowerCamelCase__   )
  
         try:
            import ftfy
            from spacy.lang.en import English
  
            lowerCAmelCase_:       List[Any]         =  English()
            lowerCAmelCase_:       Optional[int]         =  _nlp.tokenizer
            lowerCAmelCase_:       Tuple         =  ftfy.fix_text
         except ImportError:
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
            lowerCAmelCase_:       Dict         =  BasicTokenizer(do_lower_case=lowerCamelCase__   )
            lowerCAmelCase_:       str         =  None
  
         with open(lowerCamelCase__					,	encoding="utf-8"   ) as vocab_handle:
            lowerCAmelCase_:       str         =  json.load(lowerCamelCase__   )
         lowerCAmelCase_:       Union[str, Any]         =  {v: k for k, v in self.encoder.items()}
         with open(lowerCamelCase__					,	encoding="utf-8"   ) as merges_handle:
            lowerCAmelCase_:       Dict         =  merges_handle.read().split("\n"   )[1:-1]
         lowerCAmelCase_:       Dict         =  [tuple(merge.split()   ) for merge in merges]
         lowerCAmelCase_:       str         =  dict(zip(lowerCamelCase__					,	range(len(lowerCamelCase__   )   )   )   )
         lowerCAmelCase_:       List[Any]         =  {}
  
  
  
  
      @property
      def     _a						(						self   ):
         return True
  
  
  
  
      @property
      def     _a						(						self   ):
         return len(self.encoder   )
  
  
  
  
      def     _a						(						self   ):
         return dict(self.encoder					,	**self.added_tokens_encoder   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       List[str]         =  tuple(token[:-1]   ) + (token[-1] + "</w>",)
         if token in self.cache:
            return self.cache[token]
         lowerCAmelCase_:       Any         =  get_pairs(lowerCamelCase__   )
  
         if not pairs:
            return token + "</w>"
  
         while True:
            lowerCAmelCase_:       List[Any]         =  min(lowerCamelCase__					,	key=lambda lowerCamelCase__   : self.bpe_ranks.get(lowerCamelCase__					,	float("inf"   )   )   )
            if bigram not in self.bpe_ranks:
               break
            lowerCAmelCase_:       List[str]         =  bigram
            lowerCAmelCase_:       Union[str, Any]         =  []
            lowerCAmelCase_:       Optional[int]         =  0
            while i < len(lowerCamelCase__   ):
               try:
                  lowerCAmelCase_:       List[Any]         =  word.index(lowerCamelCase__					,	lowerCamelCase__   )
               except ValueError:
                  new_word.extend(word[i:]   )
                  break
               else:
                  new_word.extend(word[i:j]   )
                  lowerCAmelCase_:       Any         =  j
  
               if word[i] == first and i < len(lowerCamelCase__   ) - 1 and word[i + 1] == second:
                  new_word.append(first + second   )
                  i += 2
               else:
                  new_word.append(word[i]   )
                  i += 1
            lowerCAmelCase_:       List[Any]         =  tuple(lowerCamelCase__   )
            lowerCAmelCase_:       Optional[Any]         =  new_word
            if len(lowerCamelCase__   ) == 1:
               break
            else:
               lowerCAmelCase_:       Optional[Any]         =  get_pairs(lowerCamelCase__   )
         lowerCAmelCase_:       List[str]         =  " ".join(lowerCamelCase__   )
         if word == "\n  </w>":
            lowerCAmelCase_:       Tuple         =  "\n</w>"
         lowerCAmelCase_:       Tuple         =  word
         return word
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       str         =  []
         if self.fix_text is None:
            # Using BERT's BasicTokenizer
            lowerCAmelCase_:       Any         =  self.nlp.tokenize(lowerCamelCase__   )
            for token in text:
               split_tokens.extend(list(self.bpe(lowerCamelCase__   ).split(" "   )   )   )
         else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            lowerCAmelCase_:       Union[str, Any]         =  self.nlp(text_standardize(self.fix_text(lowerCamelCase__   )   )   )
            for token in text:
               split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
         return split_tokens
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         return self.encoder.get(lowerCamelCase__					,	self.encoder.get(self.unk_token   )   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         return self.decoder.get(lowerCamelCase__					,	self.unk_token   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Union[str, Any]         =  "".join(lowerCamelCase__   ).replace("</w>"					,	" "   ).strip()
         return out_string
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__ = None   ):
  
  
  
  
  
  
         if not os.path.isdir(lowerCamelCase__   ):
            logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
            return
         lowerCAmelCase_:       str         =  os.path.join(
             lowerCamelCase__					,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
         lowerCAmelCase_:       Any         =  os.path.join(
             lowerCamelCase__					,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )
  
         with open(lowerCamelCase__					,	"w"					,	encoding="utf-8"   ) as f:
            f.write(json.dumps(self.encoder					,	indent=2					,	sort_keys=lowerCamelCase__					,	ensure_ascii=lowerCamelCase__   ) + "\n"   )
  
         lowerCAmelCase_:       int         =  0
         with open(lowerCamelCase__					,	"w"					,	encoding="utf-8"   ) as writer:
            writer.write("#version: 0.2\n"   )
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,	key=lambda lowerCamelCase__   : kv[1]   ):
               if index != token_index:
                  logger.warning(
                      F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                      " Please check that the tokenizer is not corrupted!"   )
                  lowerCAmelCase_:       Tuple         =  token_index
               writer.write(" ".join(lowerCamelCase__   ) + "\n"   )
               index += 1
  
         return vocab_file, merge_file
style_context: import tempfile
  import unittest
  
  import numpy as np
  
  import transformers
  from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
  from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow
  
  from ...generation.test_flax_utils import FlaxGenerationTesterMixin
  from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  
  
  if is_flax_available():
   import jax
   import jax.numpy as jnp
  
   from transformers.modeling_flax_pytorch_utils import (
       convert_pytorch_state_dict_to_flax,
       load_flax_weights_in_pytorch_model,
   )
   from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel
  
  if is_torch_available():
   import torch
  
  class 				_lowercase						:
  
  
  
  
  
  
      '''simple docstring'''
  
  
  
  
      def __init__(						self					,	lowerCamelCase__					,	lowerCamelCase__=14					,	lowerCamelCase__=7					,	lowerCamelCase__=True					,	lowerCamelCase__=True					,	lowerCamelCase__=False					,	lowerCamelCase__=True					,	lowerCamelCase__=99					,	lowerCamelCase__=32					,	lowerCamelCase__=5					,	lowerCamelCase__=4					,	lowerCamelCase__=37					,	lowerCamelCase__="gelu"					,	lowerCamelCase__=0.1					,	lowerCamelCase__=0.1					,	lowerCamelCase__=512					,	lowerCamelCase__=0.0_2					,	):
         lowerCAmelCase_:       int         =  parent
         lowerCAmelCase_:       Tuple         =  batch_size
         lowerCAmelCase_:       int         =  seq_length
         lowerCAmelCase_:       Any         =  is_training
         lowerCAmelCase_:       Any         =  use_input_mask
         lowerCAmelCase_:       Optional[Any]         =  use_token_type_ids
         lowerCAmelCase_:       str         =  use_labels
         lowerCAmelCase_:       Tuple         =  vocab_size
         lowerCAmelCase_:       List[Any]         =  hidden_size
         lowerCAmelCase_:       Optional[int]         =  num_hidden_layers
         lowerCAmelCase_:       List[str]         =  num_attention_heads
         lowerCAmelCase_:       int         =  intermediate_size
         lowerCAmelCase_:       Tuple         =  hidden_act
         lowerCAmelCase_:       Dict         =  hidden_dropout_prob
         lowerCAmelCase_:       Dict         =  attention_probs_dropout_prob
         lowerCAmelCase_:       Any         =  max_position_embeddings
         lowerCAmelCase_:       Dict         =  initializer_range
         lowerCAmelCase_:       List[Any]         =  None
         lowerCAmelCase_:       List[Any]         =  vocab_size - 1
         lowerCAmelCase_:       Union[str, Any]         =  vocab_size - 1
         lowerCAmelCase_:       int         =  vocab_size - 1
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       Tuple         =  ids_tensor([self.batch_size, self.seq_length]					,	self.vocab_size   )
  
         lowerCAmelCase_:       Any         =  None
         if self.use_input_mask:
            lowerCAmelCase_:       Union[str, Any]         =  random_attention_mask([self.batch_size, self.seq_length]   )
  
         lowerCAmelCase_:       Union[str, Any]         =  GPTaConfig(
             vocab_size=self.vocab_size					,	n_embd=self.hidden_size					,	n_layer=self.num_hidden_layers					,	n_head=self.num_attention_heads					,	n_positions=self.max_position_embeddings					,	use_cache=lowerCamelCase__					,	bos_token_id=self.bos_token_id					,	eos_token_id=self.eos_token_id					,	pad_token_id=self.pad_token_id					,	)
  
         return (config, input_ids, input_mask)
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       str         =  self.prepare_config_and_inputs()
         lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Dict         =  config_and_inputs
         lowerCAmelCase_:       List[str]         =  {"input_ids": input_ids, "attention_mask": attention_mask}
         return config, inputs_dict
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       str         =  self.prepare_config_and_inputs()
  
         lowerCAmelCase_:       List[str]         =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
         lowerCAmelCase_:       Tuple         =  ids_tensor([self.batch_size, self.seq_length]					,	vocab_size=2   )
  
         return (
             config,
             input_ids,
             attention_mask,
             encoder_hidden_states,
             encoder_attention_mask,
         )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Optional[Any]         =  20
         lowerCAmelCase_:       Union[str, Any]         =  model_class_name(lowerCamelCase__   )
  
         lowerCAmelCase_:       List[str]         =  model.init_cache(input_ids.shape[0]					,	lowerCamelCase__   )
         lowerCAmelCase_:       Any         =  jnp.ones((input_ids.shape[0], max_decoder_length)					,	dtype="i4"   )
  
         lowerCAmelCase_:       Union[str, Any]         =  jnp.broadcast_to(
             jnp.arange(input_ids.shape[-1] - 1   )[None, :]					,	(input_ids.shape[0], input_ids.shape[-1] - 1)   )
         lowerCAmelCase_:       Optional[Any]         =  model(
             input_ids[:, :-1]					,	attention_mask=lowerCamelCase__					,	past_key_values=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       Any         =  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,	dtype="i4"   )
         lowerCAmelCase_:       Optional[int]         =  model(
             input_ids[:, -1:]					,	attention_mask=lowerCamelCase__					,	past_key_values=outputs_cache.past_key_values					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       List[Any]         =  model(lowerCamelCase__   )
  
         lowerCAmelCase_:       List[Any]         =  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
         self.parent.assertTrue(diff < 1E-3					,	msg=F'''Max diff is {diff}'''   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Union[str, Any]         =  20
         lowerCAmelCase_:       List[str]         =  model_class_name(lowerCamelCase__   )
  
         lowerCAmelCase_:       Any         =  jnp.concatenate(
             [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])   )]					,	axis=-1					,	)
  
         lowerCAmelCase_:       str         =  model.init_cache(input_ids.shape[0]					,	lowerCamelCase__   )
         lowerCAmelCase_:       List[Any]         =  jnp.broadcast_to(
             jnp.arange(input_ids.shape[-1] - 1   )[None, :]					,	(input_ids.shape[0], input_ids.shape[-1] - 1)   )
  
         lowerCAmelCase_:       List[Any]         =  model(
             input_ids[:, :-1]					,	attention_mask=lowerCamelCase__					,	past_key_values=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
         lowerCAmelCase_:       Dict         =  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,	dtype="i4"   )
         lowerCAmelCase_:       Dict         =  model(
             input_ids[:, -1:]					,	past_key_values=outputs_cache.past_key_values					,	attention_mask=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       Union[str, Any]         =  model(lowerCamelCase__					,	attention_mask=lowerCamelCase__   )
  
         lowerCAmelCase_:       List[str]         =  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
         self.parent.assertTrue(diff < 1E-3					,	msg=F'''Max diff is {diff}'''   )
  
  
  
  
  
  @require_flax
  class 				_lowercase						(    UpperCAmelCase__   ,     UpperCAmelCase__   ,     unittest.TestCase ):
  
  
  
  
  
  
      '''simple docstring'''
      SCREAMING_SNAKE_CASE:							Optional[int]        =						(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
      SCREAMING_SNAKE_CASE:							Union[str, Any]        =						(FlaxGPTaLMHeadModel,) if is_flax_available() else ()
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       Any         =  FlaxGPTaModelTester(self   )
  
  
  
  
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Dict         =  self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward(lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Optional[Any]         =  self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward_with_attn_mask(
                lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      @slow
      def     _a						(						self   ):
         lowerCAmelCase_:       Any         =  GPTaTokenizer.from_pretrained("gpt2"					,	pad_token="</s>"					,	padding_side="left"   )
         lowerCAmelCase_:       Optional[int]         =  tokenizer(["Hello this is a long string", "Hey"]					,	return_tensors="np"					,	padding=lowerCamelCase__					,	truncation=lowerCamelCase__   )
  
         lowerCAmelCase_:       List[Any]         =  FlaxGPTaLMHeadModel.from_pretrained("gpt2"   )
         lowerCAmelCase_:       List[str]         =  False
         lowerCAmelCase_:       int         =  model.config.eos_token_id
  
         lowerCAmelCase_:       Optional[int]         =  jax.jit(model.generate   )
  
         lowerCAmelCase_:       Any         =  jit_generate(inputs["input_ids"]					,	attention_mask=inputs["attention_mask"]   ).sequences
  
         lowerCAmelCase_:       Any         =  tokenizer.batch_decode(lowerCamelCase__					,	skip_special_tokens=lowerCamelCase__   )
  
         lowerCAmelCase_:       Dict         =  [
             "Hello this is a long string of words. I'm going to start with the first one.\n",
             "Hey, I'm not sure if I'm going to be able to do",
         ]
  
         self.assertListEqual(lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      @is_pt_flax_cross_test
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_:       Union[str, Any]         =  self.model_tester.prepare_config_and_inputs_for_common()
  
         for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__   ):
               # prepare inputs
               lowerCAmelCase_:       Any         =  self._prepare_for_class(lowerCamelCase__					,	lowerCamelCase__   )
               lowerCAmelCase_:       str         =  {k: torch.tensor(v.tolist()   ) for k, v in prepared_inputs_dict.items()}
  
               # load corresponding PyTorch class
               lowerCAmelCase_:       Dict         =  model_class.__name__[4:]  # Skip the "Flax" at the beginning
               lowerCAmelCase_:       Union[str, Any]         =  getattr(lowerCamelCase__					,	lowerCamelCase__   )
  
               lowerCAmelCase_     ,  lowerCAmelCase_:       str         =  pt_inputs["input_ids"].shape
               lowerCAmelCase_:       str         =  np.random.randint(0					,	seq_length - 1					,	size=(batch_size,)   )
               for batch_idx, start_index in enumerate(lowerCamelCase__   ):
                  lowerCAmelCase_:       List[str]         =  0
                  lowerCAmelCase_:       int         =  1
                  lowerCAmelCase_:       List[Any]         =  0
                  lowerCAmelCase_:       List[Any]         =  1
               lowerCAmelCase_:       Tuple         =  pt_model_class(lowerCamelCase__   ).eval()
               lowerCAmelCase_:       Tuple         =  model_class(lowerCamelCase__					,	dtype=jnp.floataa   )
  
               lowerCAmelCase_:       int         =  convert_pytorch_state_dict_to_flax(pt_model.state_dict()					,	lowerCamelCase__   )
               lowerCAmelCase_:       Optional[int]         =  fx_state
  
               with torch.no_grad():
                  lowerCAmelCase_:       Union[str, Any]         =  pt_model(**lowerCamelCase__   ).to_tuple()
  
               lowerCAmelCase_:       List[str]         =  fx_model(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
               with tempfile.TemporaryDirectory() as tmpdirname:
                  pt_model.save_pretrained(lowerCamelCase__   )
                  lowerCAmelCase_:       Dict         =  model_class.from_pretrained(lowerCamelCase__					,	from_pt=lowerCamelCase__   )
  
               lowerCAmelCase_:       List[Any]         =  fx_model_loaded(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(
                   len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output_loaded, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output_loaded[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
  
  
  
      @is_pt_flax_cross_test
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_:       List[str]         =  self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__   ):
               # prepare inputs
               lowerCAmelCase_:       Any         =  self._prepare_for_class(lowerCamelCase__					,	lowerCamelCase__   )
               lowerCAmelCase_:       int         =  {k: torch.tensor(v.tolist()   ) for k, v in prepared_inputs_dict.items()}
  
               # load corresponding PyTorch class
               lowerCAmelCase_:       List[str]         =  model_class.__name__[4:]  # Skip the "Flax" at the beginning
               lowerCAmelCase_:       Dict         =  getattr(lowerCamelCase__					,	lowerCamelCase__   )
  
               lowerCAmelCase_:       Dict         =  pt_model_class(lowerCamelCase__   ).eval()
               lowerCAmelCase_:       Optional[Any]         =  model_class(lowerCamelCase__					,	dtype=jnp.floataa   )
  
               lowerCAmelCase_:       Optional[Any]         =  load_flax_weights_in_pytorch_model(lowerCamelCase__					,	fx_model.params   )
               lowerCAmelCase_     ,  lowerCAmelCase_:       List[str]         =  pt_inputs["input_ids"].shape
               lowerCAmelCase_:       Optional[Any]         =  np.random.randint(0					,	seq_length - 1					,	size=(batch_size,)   )
               for batch_idx, start_index in enumerate(lowerCamelCase__   ):
                  lowerCAmelCase_:       Dict         =  0
                  lowerCAmelCase_:       List[Any]         =  1
                  lowerCAmelCase_:       Dict         =  0
                  lowerCAmelCase_:       Optional[int]         =  1
  
               # make sure weights are tied in PyTorch
               pt_model.tie_weights()
  
               with torch.no_grad():
                  lowerCAmelCase_:       Any         =  pt_model(**lowerCamelCase__   ).to_tuple()
  
               lowerCAmelCase_:       Union[str, Any]         =  fx_model(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
               with tempfile.TemporaryDirectory() as tmpdirname:
                  fx_model.save_pretrained(lowerCamelCase__   )
                  lowerCAmelCase_:       str         =  pt_model_class.from_pretrained(lowerCamelCase__					,	from_flax=lowerCamelCase__   )
  
               with torch.no_grad():
                  lowerCAmelCase_:       int         =  pt_model_loaded(**lowerCamelCase__   ).to_tuple()
  
               self.assertEqual(
                   len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
  
  
  
      @slow
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_:       Tuple         =  model_class_name.from_pretrained("gpt2"					,	from_pt=lowerCamelCase__   )
            lowerCAmelCase_:       int         =  model(np.ones((1, 1)   )   )
            self.assertIsNotNone(lowerCamelCase__   )