


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


lowerCamelCase											=       logging.get_logger(__name__)

lowerCamelCase											=       {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

lowerCamelCase											=       {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

lowerCamelCase											=       {
    'abeja/gpt-neox-japanese-2.7b': 20_48,
}




def 	a_      (    SCREAMING_SNAKE_CASE__					:      List[Any]		,			SCREAMING_SNAKE_CASE__					:      List[str]     ):
       '''simple docstring'''





       with open(SCREAMING_SNAKE_CASE__		,			'r'		,			encoding='utf-8'     ) as f:
              _lowerCamelCase       :		Dict		=json.loads(f.read()     )

       _lowerCamelCase       :		int		=collections.OrderedDict()
       _lowerCamelCase       :		Tuple		=collections.OrderedDict()
       _lowerCamelCase       :		List[Any]		=collections.OrderedDict()
       with open(SCREAMING_SNAKE_CASE__		,			'r'		,			encoding='utf-8'     ) as f:
              _lowerCamelCase       :		List[str]		=f.readlines()
       _lowerCamelCase       :		Union[str, Any]		=[[t.rstrip('\n'     )] if (t == ',' or ',' not in t) else t.rstrip('\n'     ).split(','     ) for t in token]
       for idx, b in enumerate(SCREAMING_SNAKE_CASE__     ):
              _lowerCamelCase       :		Any		=b
              _lowerCamelCase       :		Union[str, Any]		=idx
              for wd in b:
                     _lowerCamelCase       :		Any		=idx

       return vocab, raw_vocab, ids_to_tokens, emoji




class A  (   UpperCamelCase_     ):
      UpperCamelCase__							:  Tuple      =VOCAB_FILES_NAMES
      UpperCamelCase__							:  Tuple      =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__							:  Union[str, Any]      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__							:  List[Any]      =['input_ids', 'attention_mask']






      def __init__(					self						:     Optional[Any]			,   lowercase_						:     Any			,   lowercase_						:     Optional[Any]			,   lowercase_						:     Tuple="<|endoftext|>"			,   lowercase_						:     List[str]="<|endoftext|>"			,   lowercase_						:     Optional[int]="<|startoftext|>"			,   lowercase_						:     int="<|endoftext|>"			,   lowercase_						:     Optional[int]=False			,   **lowercase_						:     int			,   )					->						Union[str, Any]:
             """simple docstring"""




             super().__init__(
                 unk_token=lowercase_			,   pad_token=lowercase_			,   bos_token=lowercase_			,   eos_token=lowercase_			,   do_clean_text=lowercase_			,   **lowercase_			,   )
             if not os.path.isfile(lowercase_ ):
                    raise ValueError(
                        F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
                        ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' )
             if not os.path.isfile(lowercase_ ):
                    raise ValueError(
                        F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
                        ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' )
             _lowerCamelCase       :		str		=do_clean_text
             _lowerCamelCase	,     _lowerCamelCase	,     _lowerCamelCase	,     _lowerCamelCase       :		Union[str, Any]		=load_vocab_and_emoji(lowercase_			,   lowercase_ )
             _lowerCamelCase       :		str		=SubWordJapaneseTokenizer(
                 vocab=self.vocab			,   ids_to_tokens=self.ids_to_tokens			,   emoji=self.emoji )






      @property
      def        lowerCamelCase       (					self						:     List[str] )					->						Optional[Any]:
             """simple docstring"""




             return len(self.raw_vocab )






      def        lowerCamelCase       (					self						:     Tuple )					->						Dict:
             """simple docstring"""




             return dict(self.raw_vocab			,   **self.added_tokens_encoder )






      def        lowerCamelCase       (					self						:     Dict			,   lowercase_						:     Optional[int] )					->						str:
             """simple docstring"""




             return self.subword_tokenizer.tokenize(lowercase_			,   clean=self.do_clean_text )






      def        lowerCamelCase       (					self						:     List[Any]			,   lowercase_						:     Tuple )					->						Tuple:
             """simple docstring"""




             return self.vocab.get(lowercase_			,   self.vocab.get(self.unk_token ) )






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     Optional[int] )					->						Any:
             """simple docstring"""




             return self.subword_tokenizer.convert_id_to_token(lowercase_ )






      def        lowerCamelCase       (					self						:     List[Any]			,   lowercase_						:     Any )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=''.join(lowercase_ ).strip()
             return out_string






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     "Conversation" )					->						List[int]:
             """simple docstring"""




             _lowerCamelCase       :		Any		=[]
             for is_user, text in conversation.iter_texts():
                    input_ids.extend(self.encode(lowercase_			,   add_special_tokens=lowercase_ ) + [self.eos_token_id] )

             if len(lowercase_ ) > self.model_max_length:
                    _lowerCamelCase       :		Any		=input_ids[-self.model_max_length :]
             return input_ids






      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     str			,   lowercase_						:     Optional[str] = None )					->						Tuple[str]:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=0
             if os.path.isdir(lowercase_ ):
                    _lowerCamelCase       :		int		=os.path.join(
                        lowercase_			,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
                    _lowerCamelCase       :		List[Any]		=os.path.join(
                        lowercase_			,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file'] )
             else:
                    _lowerCamelCase       :		Union[str, Any]		=(
                        (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
                    )
                    _lowerCamelCase       :		Tuple		=(
                        (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
                    )
             with open(lowercase_			,   'w'			,   encoding='utf-8' ) as writer:
                    for token_index, token in self.ids_to_tokens.items():
                           if index != token_index:
                                  logger.warning(
                                      F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                                      ' Please check that the vocabulary is not corrupted!' )
                                  _lowerCamelCase       :		str		=token_index
                           writer.write(','.join(lowercase_ ) + '\n' )
                           index += 1
             with open(lowercase_			,   'w'			,   encoding='utf-8' ) as writer:
                    json.dump(self.emoji			,   lowercase_ )
             return vocab_file, emoji_file




class A  (   UpperCamelCase_     ):






      def __init__(					self						:     List[str]			,   lowercase_						:     Optional[Any]			,   lowercase_						:     Optional[int]			,   lowercase_						:     List[str] )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=vocab  # same as swe
             _lowerCamelCase       :		int		=ids_to_tokens  # same as bpe
             _lowerCamelCase       :		int		=emoji
             _lowerCamelCase       :		Dict		=np.max([len(lowercase_ ) for w in self.vocab.keys()] )
             _lowerCamelCase       :		Union[str, Any]		=re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)' )
             _lowerCamelCase       :		Union[str, Any]		=re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*' )
             _lowerCamelCase       :		List[Any]		=re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}' )
             _lowerCamelCase       :		Any		=re.compile(
                 R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*' )
             _lowerCamelCase       :		Tuple		=re.compile(
                 R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*' )
             _lowerCamelCase       :		Union[str, Any]		=re.compile(
                 R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*' )
             _lowerCamelCase       :		Dict		='─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
             _lowerCamelCase       :		List[Any]		='▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
             _lowerCamelCase       :		Optional[int]		=str.maketrans({k: '<BLOCK>' for k in keisen + blocks} )






      def __len__(					self						:     int )					->						str:
             """simple docstring"""




             return len(self.ids_to_tokens )






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     Tuple )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=self.content_repattera.sub('<URL>'			,   lowercase_ )
             _lowerCamelCase       :		int		=self.content_repattera.sub('<EMAIL>'			,   lowercase_ )
             _lowerCamelCase       :		Dict		=self.content_repattera.sub('<TEL>'			,   lowercase_ )
             _lowerCamelCase       :		int		=self.content_repattera.sub('<DATE>'			,   lowercase_ )
             _lowerCamelCase       :		Union[str, Any]		=self.content_repattera.sub('<DATE>'			,   lowercase_ )
             _lowerCamelCase       :		List[Any]		=self.content_repattera.sub('<PRICE>'			,   lowercase_ )
             _lowerCamelCase       :		Any		=content.translate(self.content_transa )
             while "<BLOCK><BLOCK>" in content:
                    _lowerCamelCase       :		Optional[Any]		=content.replace('<BLOCK><BLOCK>'			,   '<BLOCK>' )
             return content






      def        lowerCamelCase       (					self						:     Tuple			,   lowercase_						:     Any			,   lowercase_						:     Union[str, Any]=False )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=text.replace(' '			,   '<SP>' )
             _lowerCamelCase       :		Optional[int]		=text.replace('　'			,   '<SP>' )
             _lowerCamelCase       :		Optional[int]		=text.replace('\r\n'			,   '<BR>' )
             _lowerCamelCase       :		int		=text.replace('\n'			,   '<BR>' )
             _lowerCamelCase       :		Optional[int]		=text.replace('\r'			,   '<BR>' )
             _lowerCamelCase       :		int		=text.replace('\t'			,   '<TAB>' )
             _lowerCamelCase       :		int		=text.replace('—'			,   'ー' )
             _lowerCamelCase       :		Any		=text.replace('−'			,   'ー' )
             for k, v in self.emoji["emoji"].items():
                    if k in text:
                           _lowerCamelCase       :		List[Any]		=text.replace(lowercase_			,   lowercase_ )
             if clean:
                    _lowerCamelCase       :		Dict		=self.clean_text(lowercase_ )

             def check_simbol(lowercase_						:     List[str] ):
                    _lowerCamelCase       :		Any		=x.encode()
                    if len(lowercase_ ) == 1 and len(lowercase_ ) == 2:
                           _lowerCamelCase       :		Tuple		=(int(e[0] ) << 8) + int(e[1] )
                           if (
                               (c >= 0xc2a1 and c <= 0xc2bf)
                               or (c >= 0xc780 and c <= 0xc783)
                               or (c >= 0xcab9 and c <= 0xcbbf)
                               or (c >= 0xcc80 and c <= 0xcda2)
                           ):
                                  return True
                    return False

             def checkuae(lowercase_						:     Optional[Any] ):
                    _lowerCamelCase       :		int		=x.encode()
                    if len(lowercase_ ) == 1 and len(lowercase_ ) == 3:
                           _lowerCamelCase       :		Optional[Any]		=(int(e[0] ) << 16) + (int(e[1] ) << 8) + int(e[2] )
                           if c >= 0xe28080 and c <= 0xe2b07f:
                                  return True
                    return False

             _lowerCamelCase       :		Dict		=0
             _lowerCamelCase       :		List[str]		=[]
             while pos < len(lowercase_ ):
                    _lowerCamelCase       :		str		=min(len(lowercase_ )			,   pos + self.maxlen + 1 ) if text[pos] == '<' else pos + 3
                    _lowerCamelCase       :		Any		=[]  # (token_id, token, pos)
                    for e in range(lowercase_			,   lowercase_			,   -1 ):
                           _lowerCamelCase       :		Dict		=text[pos:e]
                           if wd in self.vocab:
                                  if wd[0] == "<" and len(lowercase_ ) > 2:
                                         _lowerCamelCase       :		List[Any]		=[(self.vocab[wd], wd, e)]
                                         break
                                  else:
                                         candidates.append((self.vocab[wd], wd, e) )
                    if len(lowercase_ ) > 0:
                           # the smallest token_id is adopted
                           _lowerCamelCase	,     _lowerCamelCase	,     _lowerCamelCase       :		List[Any]		=sorted(lowercase_			,   key=lambda lowercase_ : x[0] )[0]
                           result.append(lowercase_ )
                           _lowerCamelCase       :		int		=e
                    else:
                           _lowerCamelCase       :		List[str]		=pos + 1
                           _lowerCamelCase       :		Dict		=text[pos:end]
                           if check_simbol(lowercase_ ):
                                  result.append('<KIGOU>' )
                           elif checkuae(lowercase_ ):
                                  result.append('<U2000U2BFF>' )
                           else:
                                  for i in wd.encode('utf-8' ):
                                         result.append('<|byte%d|>' % i )
                           _lowerCamelCase       :		Tuple		=end
             return result






      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     List[Any]			,   lowercase_						:     Union[str, Any]="\n" )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=[]
             _lowerCamelCase       :		int		=[]
             _lowerCamelCase       :		Dict		=self.ids_to_tokens[index][0]
             if word[:6] == "<|byte" and word[-2:] == "|>":
                    byte_tokens.append(int(word[6:-2] ) )
             else:
                    if len(lowercase_ ) > 0:
                           words.append(bytearray(lowercase_ ).decode('utf-8'			,   errors='replace' ) )
                           _lowerCamelCase       :		str		=[]
                    if word[:7] == "<|emoji" and word[-2:] == "|>":
                           words.append(self.emoji['emoji_inv'][word] )
                    elif word == "<SP>":
                           words.append(' ' )
                    elif word == "<BR>":
                           words.append(lowercase_ )
                    elif word == "<TAB>":
                           words.append('\t' )
                    elif word == "<BLOCK>":
                           words.append('▀' )
                    elif word == "<KIGOU>":
                           words.append('ǀ' )
                    elif word == "<U2000U2BFF>":
                           words.append('‖' )
                    else:
                           words.append(lowercase_ )
             if len(lowercase_ ) > 0:
                    words.append(bytearray(lowercase_ ).decode('utf-8'			,   errors='replace' ) )
             _lowerCamelCase       :		str		=''.join(lowercase_ )
             return text






import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer




class A  :






      def __init__(					self						:     Union[str, Any]			,   lowercase_						:     Any			,   lowercase_						:     Dict=13			,   lowercase_						:     List[Any]=7			,   lowercase_						:     List[Any]=True			,   lowercase_						:     Any=True			,   lowercase_						:     Optional[int]=False			,   lowercase_						:     Tuple=True			,   lowercase_						:     List[str]=99			,   lowercase_						:     Union[str, Any]=32			,   lowercase_						:     List[Any]=5			,   lowercase_						:     Dict=4			,   lowercase_						:     List[Any]=37			,   lowercase_						:     Any="gelu"			,   lowercase_						:     Union[str, Any]=0.1			,   lowercase_						:     Tuple=0.1			,   lowercase_						:     List[Any]=512			,   lowercase_						:     Any=16			,   lowercase_						:     Optional[int]=2			,   lowercase_						:     Any=0.02			,   lowercase_						:     Dict=3			,   lowercase_						:     Union[str, Any]=4			,   lowercase_						:     Tuple=None			,   )					->						List[Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=parent
             _lowerCamelCase       :		Tuple		=batch_size
             _lowerCamelCase       :		Any		=seq_length
             _lowerCamelCase       :		int		=is_training
             _lowerCamelCase       :		int		=use_input_mask
             _lowerCamelCase       :		List[str]		=use_token_type_ids
             _lowerCamelCase       :		Dict		=use_labels
             _lowerCamelCase       :		int		=vocab_size
             _lowerCamelCase       :		Optional[Any]		=hidden_size
             _lowerCamelCase       :		Union[str, Any]		=num_hidden_layers
             _lowerCamelCase       :		Any		=num_attention_heads
             _lowerCamelCase       :		Tuple		=intermediate_size
             _lowerCamelCase       :		List[str]		=hidden_act
             _lowerCamelCase       :		int		=hidden_dropout_prob
             _lowerCamelCase       :		Optional[int]		=attention_probs_dropout_prob
             _lowerCamelCase       :		Any		=max_position_embeddings
             _lowerCamelCase       :		Optional[Any]		=type_vocab_size
             _lowerCamelCase       :		List[Any]		=type_sequence_label_size
             _lowerCamelCase       :		Union[str, Any]		=initializer_range
             _lowerCamelCase       :		Dict		=num_labels
             _lowerCamelCase       :		Optional[Any]		=num_choices
             _lowerCamelCase       :		Dict		=scope






      def        lowerCamelCase       (					self						:     Dict )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=ids_tensor([self.batch_size, self.seq_length]			,   self.vocab_size )

             _lowerCamelCase       :		Union[str, Any]		=None
             if self.use_input_mask:
                    _lowerCamelCase       :		Dict		=random_attention_mask([self.batch_size, self.seq_length] )

             _lowerCamelCase       :		str		=None
             if self.use_token_type_ids:
                    _lowerCamelCase       :		Dict		=ids_tensor([self.batch_size, self.seq_length]			,   self.type_vocab_size )

             _lowerCamelCase       :		Tuple		=None
             _lowerCamelCase       :		Optional[Any]		=None
             _lowerCamelCase       :		Optional[Any]		=None
             if self.use_labels:
                    _lowerCamelCase       :		int		=ids_tensor([self.batch_size]			,   self.type_sequence_label_size )
                    _lowerCamelCase       :		Union[str, Any]		=ids_tensor([self.batch_size, self.seq_length]			,   self.num_labels )
                    _lowerCamelCase       :		List[Any]		=ids_tensor([self.batch_size]			,   self.num_choices )

             _lowerCamelCase       :		Optional[Any]		=self.get_config()

             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






      def        lowerCamelCase       (					self						:     int )					->						Tuple:
             """simple docstring"""




             return LlamaConfig(
                 vocab_size=self.vocab_size			,   hidden_size=self.hidden_size			,   num_hidden_layers=self.num_hidden_layers			,   num_attention_heads=self.num_attention_heads			,   intermediate_size=self.intermediate_size			,   hidden_act=self.hidden_act			,   hidden_dropout_prob=self.hidden_dropout_prob			,   attention_probs_dropout_prob=self.attention_probs_dropout_prob			,   max_position_embeddings=self.max_position_embeddings			,   type_vocab_size=self.type_vocab_size			,   is_decoder=lowercase_			,   initializer_range=self.initializer_range			,   )






      def        lowerCamelCase       (					self						:     Tuple			,   lowercase_						:     Optional[int]			,   lowercase_						:     Tuple			,   lowercase_						:     List[Any]			,   lowercase_						:     str			,   lowercase_						:     Optional[Any]			,   lowercase_						:     Optional[int]			,   lowercase_						:     List[Any] )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		Dict		=LlamaModel(config=lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		List[str]		=model(lowercase_			,   attention_mask=lowercase_ )
             _lowerCamelCase       :		List[str]		=model(lowercase_ )
             self.parent.assertEqual(result.last_hidden_state.shape			,   (self.batch_size, self.seq_length, self.hidden_size) )






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     Tuple			,   lowercase_						:     Optional[int]			,   lowercase_						:     int			,   lowercase_						:     List[str]			,   lowercase_						:     Dict			,   lowercase_						:     Any			,   lowercase_						:     str			,   lowercase_						:     Dict			,   lowercase_						:     str			,   )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		str		=True
             _lowerCamelCase       :		str		=LlamaModel(lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		int		=model(
                 lowercase_			,   attention_mask=lowercase_			,   encoder_hidden_states=lowercase_			,   encoder_attention_mask=lowercase_			,   )
             _lowerCamelCase       :		str		=model(
                 lowercase_			,   attention_mask=lowercase_			,   encoder_hidden_states=lowercase_			,   )
             _lowerCamelCase       :		List[Any]		=model(lowercase_			,   attention_mask=lowercase_ )
             self.parent.assertEqual(result.last_hidden_state.shape			,   (self.batch_size, self.seq_length, self.hidden_size) )






      def        lowerCamelCase       (					self						:     Dict			,   lowercase_						:     Dict			,   lowercase_						:     List[Any]			,   lowercase_						:     Tuple			,   lowercase_						:     Dict			,   lowercase_						:     Tuple			,   lowercase_						:     str			,   lowercase_						:     List[Any]			,   lowercase_						:     List[Any]			,   lowercase_						:     Any			,   )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		int		=LlamaForCausalLM(config=lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		int		=model(lowercase_			,   attention_mask=lowercase_			,   labels=lowercase_ )
             self.parent.assertEqual(result.logits.shape			,   (self.batch_size, self.seq_length, self.vocab_size) )






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     str			,   lowercase_						:     Dict			,   lowercase_						:     str			,   lowercase_						:     Any			,   lowercase_						:     Optional[int]			,   lowercase_						:     List[Any]			,   lowercase_						:     str			,   lowercase_						:     Optional[int]			,   lowercase_						:     Optional[Any]			,   )					->						List[str]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=True
             _lowerCamelCase       :		Tuple		=True
             _lowerCamelCase       :		str		=LlamaForCausalLM(config=lowercase_ )
             model.to(lowercase_ )
             model.eval()

             # first forward pass
             _lowerCamelCase       :		int		=model(
                 lowercase_			,   attention_mask=lowercase_			,   encoder_hidden_states=lowercase_			,   encoder_attention_mask=lowercase_			,   use_cache=lowercase_			,   )
             _lowerCamelCase       :		Any		=outputs.past_key_values

             # create hypothetical multiple next token and extent to next_input_ids
             _lowerCamelCase       :		Union[str, Any]		=ids_tensor((self.batch_size, 3)			,   config.vocab_size )
             _lowerCamelCase       :		Optional[Any]		=ids_tensor((self.batch_size, 3)			,   vocab_size=2 )

             # append to next input_ids and
             _lowerCamelCase       :		Tuple		=torch.cat([input_ids, next_tokens]			,   dim=-1 )
             _lowerCamelCase       :		Optional[Any]		=torch.cat([input_mask, next_mask]			,   dim=-1 )

             _lowerCamelCase       :		Optional[int]		=model(
                 lowercase_			,   attention_mask=lowercase_			,   encoder_hidden_states=lowercase_			,   encoder_attention_mask=lowercase_			,   output_hidden_states=lowercase_			,   )['hidden_states'][0]
             _lowerCamelCase       :		Tuple		=model(
                 lowercase_			,   attention_mask=lowercase_			,   encoder_hidden_states=lowercase_			,   encoder_attention_mask=lowercase_			,   past_key_values=lowercase_			,   output_hidden_states=lowercase_			,   )['hidden_states'][0]

             # select random slice
             _lowerCamelCase       :		int		=ids_tensor((1,)			,   output_from_past.shape[-1] ).item()
             _lowerCamelCase       :		Optional[Any]		=output_from_no_past[:, -3:, random_slice_idx].detach()
             _lowerCamelCase       :		Optional[int]		=output_from_past[:, :, random_slice_idx].detach()

             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(lowercase_			,   lowercase_			,   atol=1E-3 ) )






      def        lowerCamelCase       (					self						:     Tuple )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		int		=self.prepare_config_and_inputs()
             (
                 (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     (
                 _lowerCamelCase
             )	,     
             )       :		Optional[Any]		=config_and_inputs
             _lowerCamelCase       :		Union[str, Any]		={'input_ids': input_ids, 'attention_mask': input_mask}
             return config, inputs_dict




@require_torch
class A  (   UpperCamelCase_    ,							UpperCamelCase_    ,							UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  Optional[Any]      =(LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
      UpperCamelCase__							:  Union[str, Any]      =(LlamaForCausalLM,) if is_torch_available() else ()
      UpperCamelCase__							:  List[Any]      =(
          {
              'feature-extraction': LlamaModel,
              'text-classification': LlamaForSequenceClassification,
              'text-generation': LlamaForCausalLM,
              'zero-shot': LlamaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      UpperCamelCase__							:  Optional[Any]      =False
      UpperCamelCase__							:  Tuple      =False






      def        lowerCamelCase       (					self						:     Union[str, Any] )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=LlamaModelTester(self )
             _lowerCamelCase       :		Any		=ConfigTester(self			,   config_class=lowercase_			,   hidden_size=37 )






      def        lowerCamelCase       (					self						:     List[str] )					->						Tuple:
             """simple docstring"""




             self.config_tester.run_common_tests()






      def        lowerCamelCase       (					self						:     Dict )					->						str:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		=self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*lowercase_ )






      def        lowerCamelCase       (					self						:     int )					->						Tuple:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                    _lowerCamelCase       :		Tuple		=type
                    self.model_tester.create_and_check_model(*lowercase_ )






      def        lowerCamelCase       (					self						:     Tuple )					->						List[str]:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		List[str]		=self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase       :		str		=3
             _lowerCamelCase       :		str		=input_dict['input_ids']
             _lowerCamelCase       :		int		=input_ids.ne(1 ).to(lowercase_ )
             _lowerCamelCase       :		Optional[Any]		=ids_tensor([self.model_tester.batch_size]			,   self.model_tester.type_sequence_label_size )
             _lowerCamelCase       :		Union[str, Any]		=LlamaForSequenceClassification(lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		Dict		=model(lowercase_			,   attention_mask=lowercase_			,   labels=lowercase_ )
             self.assertEqual(result.logits.shape			,   (self.model_tester.batch_size, self.model_tester.num_labels) )






      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[Any]:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		Optional[int]		=self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase       :		Optional[Any]		=3
             _lowerCamelCase       :		List[Any]		='single_label_classification'
             _lowerCamelCase       :		List[str]		=input_dict['input_ids']
             _lowerCamelCase       :		Any		=input_ids.ne(1 ).to(lowercase_ )
             _lowerCamelCase       :		List[str]		=ids_tensor([self.model_tester.batch_size]			,   self.model_tester.type_sequence_label_size )
             _lowerCamelCase       :		str		=LlamaForSequenceClassification(lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		Optional[int]		=model(lowercase_			,   attention_mask=lowercase_			,   labels=lowercase_ )
             self.assertEqual(result.logits.shape			,   (self.model_tester.batch_size, self.model_tester.num_labels) )






      def        lowerCamelCase       (					self						:     Dict )					->						int:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		List[str]		=self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase       :		int		=3
             _lowerCamelCase       :		Optional[Any]		='multi_label_classification'
             _lowerCamelCase       :		str		=input_dict['input_ids']
             _lowerCamelCase       :		Tuple		=input_ids.ne(1 ).to(lowercase_ )
             _lowerCamelCase       :		Tuple		=ids_tensor(
                 [self.model_tester.batch_size, config.num_labels]			,   self.model_tester.type_sequence_label_size ).to(torch.float )
             _lowerCamelCase       :		Optional[Any]		=LlamaForSequenceClassification(lowercase_ )
             model.to(lowercase_ )
             model.eval()
             _lowerCamelCase       :		Optional[Any]		=model(lowercase_			,   attention_mask=lowercase_			,   labels=lowercase_ )
             self.assertEqual(result.logits.shape			,   (self.model_tester.batch_size, self.model_tester.num_labels) )






      @unittest.skip('LLaMA buffers include complex numbers, which breaks this test' )
      def        lowerCamelCase       (					self						:     int )					->						str:
             """simple docstring"""




             pass






      @parameterized.expand([('linear',), ('dynamic',)] )
      def        lowerCamelCase       (					self						:     Optional[int]			,   lowercase_						:     List[Any] )					->						int:
             """simple docstring"""




             _lowerCamelCase	,     _lowerCamelCase       :		Optional[int]		=self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase       :		Union[str, Any]		=ids_tensor([1, 10]			,   config.vocab_size )
             _lowerCamelCase       :		List[str]		=ids_tensor([1, int(config.max_position_embeddings * 1.5 )]			,   config.vocab_size )

             set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase       :		Optional[int]		=LlamaModel(lowercase_ )
             original_model.to(lowercase_ )
             original_model.eval()
             _lowerCamelCase       :		List[Any]		=original_model(lowercase_ ).last_hidden_state
             _lowerCamelCase       :		Dict		=original_model(lowercase_ ).last_hidden_state

             set_seed(42 )  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase       :		Dict		={'type': scaling_type, 'factor': 10.0}
             _lowerCamelCase       :		int		=LlamaModel(lowercase_ )
             scaled_model.to(lowercase_ )
             scaled_model.eval()
             _lowerCamelCase       :		Optional[int]		=scaled_model(lowercase_ ).last_hidden_state
             _lowerCamelCase       :		Dict		=scaled_model(lowercase_ ).last_hidden_state

             # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
             # maximum sequence length, so the outputs for the short input should match.
             if scaling_type == "dynamic":
                    self.assertTrue(torch.allclose(lowercase_			,   lowercase_			,   atol=1E-5 ) )
             else:
                    self.assertFalse(torch.allclose(lowercase_			,   lowercase_			,   atol=1E-5 ) )

             # The output should be different for long inputs
             self.assertFalse(torch.allclose(lowercase_			,   lowercase_			,   atol=1E-5 ) )




@require_torch
class A  (   unittest.TestCase     ):






      @unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!' )
      @slow
      def        lowerCamelCase       (					self						:     Tuple )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=[1, 306, 4658, 278, 6593, 310, 2834, 338]
             _lowerCamelCase       :		Optional[Any]		=LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf'			,   device_map='auto' )
             _lowerCamelCase       :		int		=model(torch.tensor([input_ids] ) )
             # Expected mean on dim = -1
             _lowerCamelCase       :		int		=torch.tensor([[-6.6550, -4.1227, -4.9859, -3.2406, 0.8262, -3.0033, 1.2964, -3.3699]] )
             torch.testing.assert_close(out.mean(-1 )			,   lowercase_			,   atol=1E-2			,   rtol=1E-2 )
             # slicing logits[0, 0, 0:30]
             # fmt: off
             _lowerCamelCase       :		Dict		=torch.tensor([-12.8281, -7.4453, -0.4639, -8.0625, -7.2500, -8.0000, -6.4883, -7.7695, -7.8438, -7.0312, -6.2188, -7.1328, -1.8496, 1.9961, -8.6250, -6.7227, -12.8281, -6.9492, -7.0742, -7.7852, -7.5820, -7.9062, -6.9375, -7.9805, -8.3438, -8.1562, -8.0469, -7.6250, -7.7422, -7.3398,] )
             # fmt: on
             torch.testing.assert_close(out[0, 0, :30]			,   lowercase_			,   atol=1E-5			,   rtol=1E-5 )






      @unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!' )
      @slow
      def        lowerCamelCase       (					self						:     int )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		=[1, 306, 4658, 278, 6593, 310, 2834, 338]
             _lowerCamelCase       :		Optional[Any]		=LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-13b-hf'			,   device_map='auto' )
             _lowerCamelCase       :		List[Any]		=model(torch.tensor(lowercase_ ) )
             # Expected mean on dim = -1
             _lowerCamelCase       :		str		=torch.tensor([[-2.0622, -1.2794, -1.1638, -0.9788, -1.4603, -1.0238, -1.7893, -1.4411]] )
             torch.testing.assert_close(out.mean(-1 )			,   lowercase_			,   atol=1E-2			,   rtol=1E-2 )
             # slicing logits[0, 0, 0:30]
             # fmt: off
             _lowerCamelCase       :		List[str]		=torch.tensor([-8.1406, -8.0547, 2.7461, -1.2344, -0.1448, -1.8262, -1.0020, -1.8154, -1.6895, -1.8516, -2.3574, -0.9277, 3.7598, 6.5742, -1.2998, -0.1177, -8.1406, -2.9688, -2.9199, -3.1699, -3.5254, -2.3555, -2.7988, -3.4141, -2.8262, -4.5195, -3.3379, -3.3164, -2.7832, -3.0273] )
             # fmt: on
             torch.testing.assert_close(out[0, 0, :30]			,   lowercase_			,   atol=1E-5			,   rtol=1E-5 )






      @unittest.skip('Logits are not exactly the same, once we fix the instabalities somehow, will update!' )
      @slow
      def        lowerCamelCase       (					self						:     List[Any] )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Optional[Any]		=[1, 306, 4658, 278, 6593, 310, 2834, 338]
             _lowerCamelCase       :		str		=LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-13b-chat-hf'			,   device_map='auto' )
             _lowerCamelCase       :		int		=model(torch.tensor(lowercase_ ) )
             # Expected mean on dim = -1
             _lowerCamelCase       :		str		=torch.tensor([[-0.8562, -1.8520, -0.7551, -0.4162, -1.5161, -1.2038, -2.4823, -2.3254]] )
             torch.testing.assert_close(out.mean(-1 )			,   lowercase_			,   atol=1E-2			,   rtol=1E-2 )
             # slicing logits[0, 0, 0:30]
             # fmt: off
             _lowerCamelCase       :		Union[str, Any]		=torch.tensor([-2.2227, 4.8828, 0.9023, -0.4578, -0.7871, -0.1033, -0.6221, -0.5786, -0.7803, -1.0674, -1.2920, -0.1570, 0.8008, 2.0723, -0.9497, 0.2771, -2.2227, -0.7612, -1.4346, -1.2061, -1.6426, -0.3000, -0.7139, -1.1934, -1.8691, -1.6973, -1.5947, -1.2705, -0.3523, -0.5513] )
             # fmt: on
             torch.testing.assert_close(out.mean(-1 )			,   lowercase_			,   atol=1E-2			,   rtol=1E-2 )






      @unittest.skip(
          'Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test' )
      @slow
      def        lowerCamelCase       (					self						:     int )					->						List[Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[str]		=[1, 306, 4658, 278, 6593, 310, 2834, 338]
             _lowerCamelCase       :		Any		=LlamaForCausalLM.from_pretrained('meta-llama/Llama-2-70b-hf'			,   device_map='auto' )
             _lowerCamelCase       :		Optional[Any]		=model(torch.tensor(lowercase_ ) )

             _lowerCamelCase       :		Optional[int]		=torch.tensor(
                 [[-4.2327, -3.3360, -4.6665, -4.7631, -1.8180, -3.4170, -1.4211, -3.1810]]			,   dtype=torch.floataa )
             torch.testing.assert_close(out.mean(-1 )			,   lowercase_			,   atol=1E-2			,   rtol=1E-2 )
             # fmt: off
             _lowerCamelCase       :		int		=torch.tensor([-9.4922, -3.9551, 1.7998, -5.6758, -5.1055, -5.8984, -4.8320, -6.8086, -6.5391, -5.6172, -5.5820, -5.5352, 1.7881, 3.6289, -6.5117, -3.4785, -9.5000, -6.0352, -6.8125, -6.0195, -6.6836, -5.4727, -6.2812, -6.0391, -7.3398, -7.4297, -7.4844, -6.5820, -5.8789, -5.5312] )
             # fmt: on
             torch.testing.assert_close(out[0, 0, :30]			,   lowercase_			,   atol=1E-5			,   rtol=1E-5 )






      @unittest.skip('Model is curently gated' )
      @slow
      def        lowerCamelCase       (					self						:     str )					->						Optional[int]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		='Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the "princi'
             _lowerCamelCase       :		Union[str, Any]		='Simply put, the theory of relativity states that '
             _lowerCamelCase       :		int		=LlamaTokenizer.from_pretrained('meta-llama/Llama-2-13b-chat-hf' )
             _lowerCamelCase       :		str		=tokenizer.encode(lowercase_			,   return_tensors='pt' )
             _lowerCamelCase       :		List[Any]		=LlamaForCausalLM.from_pretrained(
                 'meta-llama/Llama-2-13b-chat-hf'			,   device_map='sequential'			,   use_safetensors=lowercase_ )

             # greedy generation outputs
             _lowerCamelCase       :		str		=model.generate(lowercase_			,   max_new_tokens=64			,   top_p=lowercase_			,   temperature=1			,   do_sample=lowercase_ )
             _lowerCamelCase       :		Tuple		=tokenizer.decode(generated_ids[0]			,   skip_special_tokens=lowercase_ )
             self.assertEqual(lowercase_			,   lowercase_ )


