import{s as Nt,o as Bt,n as tt}from"../chunks/scheduler.25b97de1.js";import{S as Gt,i as kt,g as l,s as n,r as c,A as Lt,h as r,f as t,c as s,j as V,u as m,x as f,k as Z,y as d,a,v as p,d as h,t as M,w as u}from"../chunks/index.d9030fc9.js";import{T as zt}from"../chunks/Tip.baa67368.js";import{D as B}from"../chunks/Docstring.e257edda.js";import{C as F}from"../chunks/CodeBlock.e6cd0d95.js";import{E as Zt}from"../chunks/ExampleCodeBlock.20db4b6e.js";import{H as C,E as Wt}from"../chunks/EditOnGithub.91d95064.js";function At(G){let g,J="Example:",T,y,U;return y=new F({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYXZhTmV4dFZpZGVvRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwTGxhdmFOZXh0VmlkZW9Db25maWclMkMlMjBDTElQVmlzaW9uQ29uZmlnJTJDJTIwTGxhbWFDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwQ0xJUC12aXNpb24lMjBjb25maWclMEF2aXNpb25fY29uZmlnJTIwJTNEJTIwQ0xJUFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMExsYW1hJTIwY29uZmlnJTBBdGV4dF9jb25maWclMjAlM0QlMjBMbGFtYUNvbmZpZygpJTBBJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMExsYXZhTmV4dFZpZGVvQ29uZmlnKHZpc2lvbl9jb25maWclMkMlMjB0ZXh0X2NvbmZpZyklMEElMEFtb2RlbCUyMCUzRCUyMExsYXZhTmV4dFZpZGVvRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uKGNvbmZpZ3VyYXRpb24pJTBBJTBBJTIzJTIwQWNjZXNzaW5nJTIwdGhlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMG1vZGVsLmNvbmZpZw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlavaNextVideoForConditionalGeneration, LlavaNextVideoConfig, CLIPVisionConfig, LlamaConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a CLIP-vision config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = CLIPVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Llama config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = LlamaConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = LlavaNextVideoConfig(vision_config, text_config)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = LlavaNextVideoForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){g=l("p"),g.textContent=J,T=n(),c(y.$$.fragment)},l(i){g=r(i,"P",{"data-svelte-h":!0}),f(g)!=="svelte-11lpom8"&&(g.textContent=J),T=s(i),m(y.$$.fragment,i)},m(i,v){a(i,g,v),a(i,T,v),p(y,i,v),U=!0},p:tt,i(i){U||(h(y.$$.fragment,i),U=!0)},o(i){M(y.$$.fragment,i),U=!1},d(i){i&&(t(g),t(T)),u(y,i)}}}function Rt(G){let g,J=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){g=l("p"),g.innerHTML=J},l(T){g=r(T,"P",{"data-svelte-h":!0}),f(g)!=="svelte-fincs2"&&(g.innerHTML=J)},m(T,y){a(T,g,y)},p:tt,d(T){T&&t(g)}}}function Xt(G){let g,J="Example:",T,y,U;return y=new F({props:{code:" ",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> av
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, LlavaNextVideoForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">read_video_pyav</span>(<span class="hljs-params">container, indices</span>):
<span class="hljs-meta">... </span>    <span class="hljs-string">&#x27;&#x27;&#x27;
<span class="hljs-meta">... </span>    Decode the video with PyAV decoder.
<span class="hljs-meta">... </span>    Args:
<span class="hljs-meta">... </span>        container (\`av.container.input.InputContainer\`): PyAV container.
<span class="hljs-meta">... </span>        indices (\`List[int]\`): List of frame indices to decode.
<span class="hljs-meta">... </span>    Returns:
<span class="hljs-meta">... </span>        result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
<span class="hljs-meta">... </span>    &#x27;&#x27;&#x27;</span>
<span class="hljs-meta">... </span>    frames = []
<span class="hljs-meta">... </span>    container.seek(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>    start_index = indices[<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>    end_index = indices[-<span class="hljs-number">1</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, frame <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(container.decode(video=<span class="hljs-number">0</span>)):
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> i &gt; end_index:
<span class="hljs-meta">... </span>            <span class="hljs-keyword">break</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> i &gt;= start_index <span class="hljs-keyword">and</span> i <span class="hljs-keyword">in</span> indices:
<span class="hljs-meta">... </span>            frames.append(frame)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> np.stack([x.to_ndarray(<span class="hljs-built_in">format</span>=<span class="hljs-string">&quot;rgb24&quot;</span>) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> frames])

<span class="hljs-meta">&gt;&gt;&gt; </span>model = LlavaNextVideoForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>, device_map=<span class="hljs-string">&quot;auto)
&gt;&gt;&gt; processor = AutoProcessor.from_pretrained(&quot;</span>llava-hf/LLaVA-NeXT-Video-7B-h<span class="hljs-string">f&quot;)

&gt;&gt;&gt; prompt = &quot;</span>USER: &lt;video&gt;\\nWhy <span class="hljs-keyword">is</span> this video funny? ASSISTANT:<span class="hljs-string">&quot;
&gt;&gt;&gt; video_path = hf_hub_download(repo_id=&quot;</span>raushan-testing-hf/videos-test<span class="hljs-string">&quot;, filename=&quot;</span>sample_demo_1.mp4<span class="hljs-string">&quot;, repo_type=&quot;</span>dataset<span class="hljs-string">&quot;)
&gt;&gt;&gt; container = av.open(video_path)

&gt;&gt;&gt; # sample uniformly 8 frames from the video (model was trained with 32 frames per video, but this video is short)
&gt;&gt;&gt; total_frames = container.streams.video[0].frames
&gt;&gt;&gt; indices = np.arange(0, total_frames, total_frames / 8).astype(int)
&gt;&gt;&gt; clip = read_video_pyav(container, indices)
&gt;&gt;&gt; inputs_video = processor(text=prompt, videos=clip, return_tensors=&quot;</span>pt<span class="hljs-string">&quot;).to(model.device)

&gt;&gt;&gt; # load an image to generate from an image
&gt;&gt;&gt; prompt = &quot;</span>USER:&lt;image&gt;\\nWhat <span class="hljs-keyword">is</span> shown <span class="hljs-keyword">in</span> this image? ASSISTANT:<span class="hljs-string">&quot;
&gt;&gt;&gt; url = &quot;</span>https://www.ilankelman.org/stopsigns/australia.jpg<span class="hljs-string">&quot;
&gt;&gt;&gt; image = Image.open(requests.get(url, stream=True).raw)
&gt;&gt;&gt; inputs_image = processor(text=prompt, images=image, return_tensors=&quot;</span>pt<span class="hljs-string">&quot;).to(model.device)

&gt;&gt;&gt; # Generate from video
&gt;&gt;&gt; generate_ids = model.generate(**inputs_video, max_length=50)
&gt;&gt;&gt; processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
&quot;</span>USER:\\nWhy <span class="hljs-keyword">is</span> this video funny? ASSISTANT: The humor <span class="hljs-keyword">in</span> this video comes <span class="hljs-keyword">from</span> the unexpected <span class="hljs-keyword">and</span> endearing sight of a baby wearing glasses <span class="hljs-keyword">and</span> (...)<span class="hljs-string">&quot;

&gt;&gt;&gt; # Generate from image
&gt;&gt;&gt; generate_ids = model.generate(**inputs_image, max_length=30)
&gt;&gt;&gt; processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
&quot;</span>USER: \\nWhat<span class="hljs-string">&#x27;s the content of the image? ASSISTANT: The image shows a red stop sign on a pole, with a traditional Chinese archway (...)&quot;</span>`,wrap:!1}}),{c(){g=l("p"),g.textContent=J,T=n(),c(y.$$.fragment)},l(i){g=r(i,"P",{"data-svelte-h":!0}),f(g)!=="svelte-11lpom8"&&(g.textContent=J),T=s(i),m(y.$$.fragment,i)},m(i,v){a(i,g,v),a(i,T,v),p(y,i,v),U=!0},p:tt,i(i){U||(h(y.$$.fragment,i),U=!0)},o(i){M(y.$$.fragment,i),U=!1},d(i){i&&(t(g),t(T)),u(y,i)}}}function $t(G){let g,J,T,y,U,i,v,Ye,Q,at='The LLaVa-NeXT-Video model was proposed in <a href="https://llava-vl.github.io/blog/2024-04-30-llava-next-video/" rel="nofollow">LLaVA-NeXT: A Strong Zero-shot Video Understanding Model</a> by Yuanhan Zhang, Bo Li, Haotian Liu, Yong Jae Lee, Liangke Gui, Di Fu, Jiashi Feng, Ziwei Liu, Chunyuan Li. LLaVa-NeXT-Video improves upon <a href="llava_next">LLaVa-NeXT</a> by fine-tuning on a mix if video and image dataset thus increasing the model’s performance on videos.',Pe,E,nt='<a href="llava_next">LLaVA-NeXT</a> surprisingly has strong performance in understanding video content in zero-shot fashion with the AnyRes technique that it uses. The AnyRes technique naturally represents a high-resolution image into multiple images. This technique is naturally generalizable to represent videos because videos can be considered as a set of frames (similar to a set of images in LLaVa-NeXT). The current version of LLaVA-NeXT makes use of AnyRes and trains with supervised fine-tuning (SFT) on top of LLaVA-Next on video data to achieves better video understanding capabilities.The model is a current SOTA among open-source models on <a href="https://arxiv.org/abs/2405.21075" rel="nofollow">VideoMME bench</a>.',He,S,st="The introduction from the blog is the following:",De,Y,lt="On January 30, 2024, we released LLaVA-NeXT, an open-source Large Multimodal Model (LMM) that has been trained exclusively on text-image data. With the proposed AnyRes technique, it boosts capabilities in reasoning, OCR, and world knowledge, demonstrating remarkable performance across a spectrum of image-based multimodal understanding tasks, and even exceeding Gemini-Pro on several image benchmarks, e.g. MMMU and MathVista.",Ke,P,rt="**In today’s exploration, we delve into the performance of LLaVA-NeXT within the realm of video understanding tasks. We reveal that LLaVA-NeXT surprisingly has strong performance in understanding video content. The current version of LLaVA-NeXT for videos has several improvements:",Oe,H,it="<li>Zero-shot video representation capabilities with AnyRes: The AnyRes technique naturally represents a high-resolution image into multiple images that a pre-trained VIT is able to digest, and forms them into a concantenated sequence. This technique is naturally generalizable to represent videos (consisting of multiple frames), allowing the image-only-trained LLaVA-Next model to perform surprisingly well on video tasks. Notably, this is the first time that LMMs show strong zero-shot modality transfer ability.</li> <li>Inference with length generalization improves on longer videos. The linear scaling technique enables length generalization, allowing LLaVA-NeXT to effectively handle long-video beyond the limitation of the “max_token_length” of the LLM.</li> <li>Strong video understanding ability. (1) LLaVA-Next-Image, which combines the above two techniques, yields superior zero-shot performance than open-source LMMs tuned on videos. (2) LLaVA-Next-Video, further supervised fine-tuning (SFT) LLaVA-Next-Image on video data, achieves better video understanding capabilities compared to LLaVA-Next-Image. (3) LLaVA-Next-Video-DPO, which aligns the model response with AI feedback using direct preference optimization (DPO), showing significant performance boost.</li> <li>Efficient deployment and inference with SGLang. It allows 5x faster inference on video tasks, allowing more scalable serving such as million-level video re-captioning. See instructions in our repo.**</li>",eo,D,dt=`This model was contributed by <a href="https://huggingface.co/RaushanTurganbay" rel="nofollow">RaushanTurganbay</a>.
The original code can be found <a href="https://github.com/LLaVA-VL/LLaVA-NeXT/tree/inference" rel="nofollow">here</a>.`,oo,K,to,O,ct="<li><p>We advise users to use <code>padding_side=&quot;left&quot;</code> when computing batched generation as it leads to more accurate results. Simply make sure to call <code>processor.tokenizer.padding_side = &quot;left&quot;</code> before generating.</p></li> <li><p>Note that each checkpoint has been trained with a specific prompt format, depending on which large language model (LLM) was used. You can use tokenizer’s <code>apply_chat_template</code> to format your prompts correctly. Below is an example of how to do that.</p></li>",ao,ee,mt='We will use <a href="https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-hf" rel="nofollow">LLaVA-NeXT-Video-7B-hf</a> and a conversation history of videos and images. Each content field has to be a list of dicts, as follows:',no,oe,so,te,lo,ae,ro,ne,pt="The model can accept both images and videos as input. Here’s an example code for inference in half-precision (<code>torch.float16</code>):",io,se,co,le,mo,re,ht="The model can also generate from an interleaved image-video inputs. However note, that it was not trained in interleaved image-video setting which might affect the performance. Below is an example usage for mixed media input, add the following lines to the above code snippet:",po,ie,ho,de,Mo,ce,uo,me,Mt="The model can be loaded in lower bits, significantly reducing memory burden while maintaining the performance of the original model. This allows for efficient deployment on resource-constrained cases.",go,pe,ut='First make sure to install bitsandbytes by running <code>pip install bitsandbytes</code> and to have access to a CUDA compatible GPU device. Load the quantized model by simply adding <a href="../main_classes/quantization#transformers.BitsAndBytesConfig"><code>BitsAndBytesConfig</code></a> as shown below:',fo,he,yo,Me,To,ue,gt='Additionally, we can greatly speed-up model inference by using <a href="../perf_train_gpu_one.md#flash-attention-2">Flash Attention</a>, which is a faster implementation of the attention mechanism used inside the model.',Uo,ge,ft="First, make sure to install the latest version of Flash Attention 2:",vo,fe,_o,ye,yt='Also, you should have a hardware that is compatible with Flash-Attention 2. Read more about it in the official documentation of the <a href="https://github.com/Dao-AILab/flash-attention" rel="nofollow">flash attention repository</a>. FlashAttention-2 can only be used when a model is loaded in <code>torch.float16</code> or <code>torch.bfloat16</code>.',Jo,Te,Tt="To load and run a model using Flash Attention-2, simply add <code>attn_implementation=&quot;flash_attention_2&quot;</code> when loading the model as follows:",wo,Ue,bo,ve,jo,x,_e,Wo,Le,Ut=`This is the configuration class to store the configuration of a <a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoForConditionalGeneration">LlavaNextVideoForConditionalGeneration</a>. It is used to instantiate an
Llava-NeXT model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the <a href="https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-hf" rel="nofollow">llava-hf/LLaVA-NeXT-Video-7B-hf</a>
model.
Configuration objects inherit from <a href="/docs/transformers/pr_30910/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_30910/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Ao,k,Co,Je,Io,_,we,Ro,ze,vt=`Constructs a LLaVa-NeXT-Video processor which wraps a LLaVa-NeXT image processor, LLaVa-NeXT-Video video processor and
a LLaMa tokenizer into a single processor.`,Xo,We,_t=`<a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoProcessor">LlavaNextVideoProcessor</a> offers all the functionalities of <a href="/docs/transformers/pr_30910/en/model_doc/llava_next#transformers.LlavaNextImageProcessor">LlavaNextImageProcessor</a>, <a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoImageProcessor">LlavaNextVideoImageProcessor</a> and
<a href="/docs/transformers/pr_30910/en/model_doc/llama#transformers.LlamaTokenizerFast">LlamaTokenizerFast</a>. See the <code>__call__()</code> and <a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoProcessor.decode">decode()</a> for more information.`,$o,L,be,qo,Ae,Jt=`This method forwards all its arguments to LlamaTokenizerFast’s <a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.batch_decode">batch_decode()</a>. Please
refer to the docstring of this method for more information.`,Fo,z,je,Qo,Re,wt=`This method forwards all its arguments to LlamaTokenizerFast’s <a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.decode">decode()</a>. Please refer to
the docstring of this method for more information.`,xo,Ce,Vo,w,Ie,Eo,Xe,bt='Constructs a LLaVa-NeXT-Video video processor. Based on <a href="/docs/transformers/pr_30910/en/model_doc/clip#transformers.CLIPImageProcessor">CLIPImageProcessor</a> with incorporation of processing each video frame.',So,$e,xe,Yo,W,Ve,Po,qe,jt=`Resize an image. The shortest edge of the image is resized to size[“shortest_edge”], with the longest edge
resized to keep the input aspect ratio.`,Zo,Ze,No,b,Ne,Ho,Fe,Ct=`The LLAVA-NeXT model which consists of a vision backbone and a language model.
This model inherits from <a href="/docs/transformers/pr_30910/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,Do,Qe,It=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,Ko,I,Be,Oo,Ee,xt='The <a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoForConditionalGeneration">LlavaNextVideoForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.',et,A,ot,R,Bo,Ge,Go,Se,ko;return U=new C({props:{title:"LLaVa-NeXT-Video",local:"llava-next-video",headingTag:"h1"}}),v=new C({props:{title:"Overview",local:"overview",headingTag:"h2"}}),K=new C({props:{title:"Usage tips",local:"usage-tips",headingTag:"h2"}}),oe=new F({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYXZhTmV4dFZpZGVvUHJvY2Vzc29yJTBBJTBBcHJvY2Vzc29yJTIwJTNEJTIwTGxhdmFOZXh0VmlkZW9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMmxsYXZhLWhmJTJGTExhVkEtTmVYVC1WaWRlby03Qi1oZiUyMiklMEElMEFjb252ZXJzYXRpb24lMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIyc3lzdGVtJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMkElMjBjaGF0JTIwYmV0d2VlbiUyMGElMjBjdXJpb3VzJTIwaHVtYW4lMjBhbmQlMjBhbiUyMGFydGlmaWNpYWwlMjBpbnRlbGxpZ2VuY2UlMjBhc3Npc3RhbnQuJTIwVGhlJTIwYXNzaXN0YW50JTIwZ2l2ZXMlMjBoZWxwZnVsJTJDJTIwZGV0YWlsZWQlMkMlMjBhbmQlMjBwb2xpdGUlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBodW1hbidzJTIwcXVlc3Rpb25zLiUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMldoYXQlRTIlODAlOTlzJTIwc2hvd24lMjBpbiUyMHRoaXMlMjBpbWFnZSUzRiUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJpbWFnZSUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJhc3Npc3RhbnQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb250ZW50JTIyJTNBJTIwJTVCJTdCJTIydHlwZSUyMiUzQSUyMCUyMnRleHQlMjIlMkMlMjAlMjJ0ZXh0JTIyJTNBJTIwJTIyVGhpcyUyMGltYWdlJTIwc2hvd3MlMjBhJTIwcmVkJTIwc3RvcCUyMHNpZ24uJTIyJTdEJTJDJTVEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb250ZW50JTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMnRleHQlMjIlMkMlMjAlMjJ0ZXh0JTIyJTNBJTIwJTIyV2h5JTIwaXMlMjB0aGlzJTIwdmlkZW8lMjBmdW5ueSUzRiUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ2aWRlbyUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSU1RCUwQSUwQXRleHRfcHJvbXB0JTIwJTNEJTIwcHJvY2Vzc29yLmFwcGx5X2NoYXRfdGVtcGxhdGUoY29udmVyc2F0aW9uJTJDJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSklMEElMEElMjMlMjBOb3RlJTIwdGhhdCUyMHRoZSUyMHRlbXBsYXRlJTIwc2ltcGx5JTIwZm9ybWF0cyUyMHlvdXIlMjBwcm9tcHQlMkMlMjB5b3UlMjBzdGlsbCUyMGhhdmUlMjB0byUyMHRva2VuaXplJTIwaXQlMjBhbmQlMjBvYnRhaW4lMjBwaXhlbCUyMHZhbHVlcyUyMGZvciUyMHlvdXIlMjB2aXN1YWxzJTBBcHJpbnQodGV4dF9wcm9tcHQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlavaNextVideoProcessor

processor = LlavaNextVideoProcessor.from_pretrained(<span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>)

conversation = [
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human&#x27;s questions.&quot;</span>},
            ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;What’s shown in this image?&quot;</span>},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
            ],
    },
    {
        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;This image shows a red stop sign.&quot;</span>},]
    },
    {

        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;Why is this video funny?&quot;</span>},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;video&quot;</span>},
            ],
    },
]

text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Note that the template simply formats your prompt, you still have to tokenize it and obtain pixel values for your visuals</span>
<span class="hljs-built_in">print</span>(text_prompt)`,wrap:!1}}),te=new C({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),ae=new C({props:{title:"Single Media Mode",local:"single-media-mode",headingTag:"h3"}}),se=new F({props:{code:"aW1wb3J0JTIwYXYlMEFpbXBvcnQlMjB0b3JjaCUwQWltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBMbGF2YU5leHRWaWRlb0ZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUyQyUyMExsYXZhTmV4dFZpZGVvUHJvY2Vzc29yJTBBJTBBZGVmJTIwcmVhZF92aWRlb19weWF2KGNvbnRhaW5lciUyQyUyMGluZGljZXMpJTNBJTBBJTIwJTIwJTIwJTIwJycnJTBBJTIwJTIwJTIwJTIwRGVjb2RlJTIwdGhlJTIwdmlkZW8lMjB3aXRoJTIwUHlBViUyMGRlY29kZXIuJTBBJTIwJTIwJTIwJTIwQXJncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbnRhaW5lciUyMCglNjBhdi5jb250YWluZXIuaW5wdXQuSW5wdXRDb250YWluZXIlNjApJTNBJTIwUHlBViUyMGNvbnRhaW5lci4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbmRpY2VzJTIwKCU2MExpc3QlNUJpbnQlNUQlNjApJTNBJTIwTGlzdCUyMG9mJTIwZnJhbWUlMjBpbmRpY2VzJTIwdG8lMjBkZWNvZGUuJTBBJTIwJTIwJTIwJTIwUmV0dXJucyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdCUyMChucC5uZGFycmF5KSUzQSUyMG5wJTIwYXJyYXklMjBvZiUyMGRlY29kZWQlMjBmcmFtZXMlMjBvZiUyMHNoYXBlJTIwKG51bV9mcmFtZXMlMkMlMjBoZWlnaHQlMkMlMjB3aWR0aCUyQyUyMDMpLiUwQSUyMCUyMCUyMCUyMCcnJyUwQSUyMCUyMCUyMCUyMGZyYW1lcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGNvbnRhaW5lci5zZWVrKDApJTBBJTIwJTIwJTIwJTIwc3RhcnRfaW5kZXglMjAlM0QlMjBpbmRpY2VzJTVCMCU1RCUwQSUyMCUyMCUyMCUyMGVuZF9pbmRleCUyMCUzRCUyMGluZGljZXMlNUItMSU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGklMkMlMjBmcmFtZSUyMGluJTIwZW51bWVyYXRlKGNvbnRhaW5lci5kZWNvZGUodmlkZW8lM0QwKSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGklMjAlM0UlMjBlbmRfaW5kZXglM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBicmVhayUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwaSUyMCUzRSUzRCUyMHN0YXJ0X2luZGV4JTIwYW5kJTIwaSUyMGluJTIwaW5kaWNlcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZyYW1lcy5hcHBlbmQoZnJhbWUpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbnAuc3RhY2soJTVCeC50b19uZGFycmF5KGZvcm1hdCUzRCUyMnJnYjI0JTIyKSUyMGZvciUyMHglMjBpbiUyMGZyYW1lcyU1RCklMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwbW9kZWwlMjBpbiUyMGhhbGYtcHJlY2lzaW9uJTBBbW9kZWwlMjAlM0QlMjBMbGF2YU5leHRWaWRlb0ZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTIybGxhdmEtaGYlMkZMTGFWQS1OZVhULVZpZGVvLTdCLWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwTGxhdmFOZXh0VmlkZW9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMmxsYXZhLWhmJTJGTExhVkEtTmVYVC1WaWRlby03Qi1oZiUyMiklMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwdmlkZW8lMjBhcyUyMGFuJTIwbnAuYXJyYXklMkMlMjBzYW1wbGluZyUyMHVuaWZvcm1seSUyMDglMjBmcmFtZXMlMjAoY2FuJTIwc2FtcGxlJTIwbW9yZSUyMGZvciUyMGxvbmdlciUyMHZpZGVvcyklMEF2aWRlb19wYXRoJTIwJTNEJTIwaGZfaHViX2Rvd25sb2FkKHJlcG9faWQlM0QlMjJyYXVzaGFuLXRlc3RpbmctaGYlMkZ2aWRlb3MtdGVzdCUyMiUyQyUyMGZpbGVuYW1lJTNEJTIyc2FtcGxlX2RlbW9fMS5tcDQlMjIlMkMlMjByZXBvX3R5cGUlM0QlMjJkYXRhc2V0JTIyKSUwQWNvbnRhaW5lciUyMCUzRCUyMGF2Lm9wZW4odmlkZW9fcGF0aCklMEF0b3RhbF9mcmFtZXMlMjAlM0QlMjBjb250YWluZXIuc3RyZWFtcy52aWRlbyU1QjAlNUQuZnJhbWVzJTBBaW5kaWNlcyUyMCUzRCUyMG5wLmFyYW5nZSgwJTJDJTIwdG90YWxfZnJhbWVzJTJDJTIwdG90YWxfZnJhbWVzJTIwJTJGJTIwOCkuYXN0eXBlKGludCklMEF2aWRlbyUyMCUzRCUyMHJlYWRfdmlkZW9fcHlhdihjb250YWluZXIlMkMlMjBpbmRpY2VzKSUwQSUwQWNvbnZlcnNhdGlvbiUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMldoeSUyMGlzJTIwdGhpcyUyMHZpZGVvJTIwZnVubnklM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydmlkZW8lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElNUQlMEElMEFwcm9tcHQlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZShjb252ZXJzYXRpb24lMkMlMjBhZGRfZ2VuZXJhdGlvbl9wcm9tcHQlM0RUcnVlKSUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEcHJvbXB0JTJDJTIwdmlkZW9zJTNEdmlkZW8lMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW91dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0Q2MCklMEFwcm9jZXNzb3IuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlJTJDJTIwY2xlYW5fdXBfdG9rZW5pemF0aW9uX3NwYWNlcyUzRFRydWUp",highlighted:`<span class="hljs-keyword">import</span> av
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlavaNextVideoForConditionalGeneration, LlavaNextVideoProcessor

<span class="hljs-keyword">def</span> <span class="hljs-title function_">read_video_pyav</span>(<span class="hljs-params">container, indices</span>):
    <span class="hljs-string">&#x27;&#x27;&#x27;
    Decode the video with PyAV decoder.
    Args:
        container (\`av.container.input.InputContainer\`): PyAV container.
        indices (\`List[int]\`): List of frame indices to decode.
    Returns:
        result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
    &#x27;&#x27;&#x27;</span>
    frames = []
    container.seek(<span class="hljs-number">0</span>)
    start_index = indices[<span class="hljs-number">0</span>]
    end_index = indices[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">for</span> i, frame <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(container.decode(video=<span class="hljs-number">0</span>)):
        <span class="hljs-keyword">if</span> i &gt; end_index:
            <span class="hljs-keyword">break</span>
        <span class="hljs-keyword">if</span> i &gt;= start_index <span class="hljs-keyword">and</span> i <span class="hljs-keyword">in</span> indices:
            frames.append(frame)
    <span class="hljs-keyword">return</span> np.stack([x.to_ndarray(<span class="hljs-built_in">format</span>=<span class="hljs-string">&quot;rgb24&quot;</span>) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> frames])

<span class="hljs-comment"># Load the model in half-precision</span>
model = LlavaNextVideoForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
processor = LlavaNextVideoProcessor.from_pretrained(<span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>)

<span class="hljs-comment"># Load the video as an np.array, sampling uniformly 8 frames (can sample more for longer videos)</span>
video_path = hf_hub_download(repo_id=<span class="hljs-string">&quot;raushan-testing-hf/videos-test&quot;</span>, filename=<span class="hljs-string">&quot;sample_demo_1.mp4&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)
container = av.<span class="hljs-built_in">open</span>(video_path)
total_frames = container.streams.video[<span class="hljs-number">0</span>].frames
indices = np.arange(<span class="hljs-number">0</span>, total_frames, total_frames / <span class="hljs-number">8</span>).astype(<span class="hljs-built_in">int</span>)
video = read_video_pyav(container, indices)

conversation = [
    {

        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;Why is this video funny?&quot;</span>},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;video&quot;</span>},
            ],
    },
]

prompt = processor.apply_chat_template(conversation, add_generation_prompt=<span class="hljs-literal">True</span>)
inputs = processor(text=prompt, videos=video, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

out = model.generate(**inputs, max_new_tokens=<span class="hljs-number">60</span>)
processor.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">True</span>)`,wrap:!1}}),le=new C({props:{title:"Mixed Media Mode",local:"mixed-media-mode",headingTag:"h3"}}),ie=new F({props:{code:"ZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEElMjMlMjBHZW5lcmF0ZSUyMGZyb20lMjBpbWFnZSUyMGFuZCUyMHZpZGVvJTIwbWl4ZWQlMjBpbnB1dHMlMEElMjMlMjBMb2FkJTIwYW5kJTIwaW1hZ2UlMjBhbmQlMjB3cml0ZSUyMGElMjBuZXclMjBwcm9tcHQlMEF1cmwlMjAlM0QlMjAlMjJodHRwJTNBJTJGJTJGaW1hZ2VzLmNvY29kYXRhc2V0Lm9yZyUyRnZhbDIwMTclMkYwMDAwMDAwMzk3NjkuanBnJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBY29udmVyc2F0aW9uJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb250ZW50JTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMnRleHQlMjIlMkMlMjAlMjJ0ZXh0JTIyJTNBJTIwJTIySG93JTIwbWFueSUyMGNhdHMlMjBhcmUlMjB0aGVyZSUyMGluJTIwdGhlJTIwaW1hZ2UlM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyaW1hZ2UlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlN0IlMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIyYXNzaXN0YW50JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMlRoZXJlJTIwYXJlJTIwdHdvJTIwY2F0cyUyMiU3RCU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIydGV4dCUyMiUzQSUyMCUyMldoeSUyMGlzJTIwdGhpcyUyMHZpZGVvJTIwZnVubnklM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydmlkZW8lMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElNUQlMEFwcm9tcHQlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZShjb252ZXJzYXRpb24lMkMlMjBhZGRfZ2VuZXJhdGlvbl9wcm9tcHQlM0RUcnVlKSUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEcHJvbXB0JTJDJTIwaW1hZ2VzJTNEaW1hZ2UlMkMlMjB2aWRlb3MlM0RjbGlwJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMEdlbmVyYXRlJTBBZ2VuZXJhdGVfaWRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbGVuZ3RoJTNENTApJTBBcHJvY2Vzc29yLmJhdGNoX2RlY29kZShnZW5lcmF0ZV9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSUyQyUyMGNsZWFuX3VwX3Rva2VuaXphdGlvbl9zcGFjZXMlM0RUcnVlKSUwQQ==",highlighted:`<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> requests

<span class="hljs-comment"># Generate from image and video mixed inputs</span>
<span class="hljs-comment"># Load and image and write a new prompt</span>
url = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)
conversation = [
    {

        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;How many cats are there in the image?&quot;</span>},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
            ],
    },
    {

        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [{<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;There are two cats&quot;</span>}],
    },
    {

        <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
        <span class="hljs-string">&quot;content&quot;</span>: [
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;Why is this video funny?&quot;</span>},
            {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;video&quot;</span>},
            ],
    },
]
prompt = processor.apply_chat_template(conversation, add_generation_prompt=<span class="hljs-literal">True</span>)
inputs = processor(text=prompt, images=image, videos=clip, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Generate</span>
generate_ids = model.generate(**inputs, max_length=<span class="hljs-number">50</span>)
processor.batch_decode(generate_ids, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">True</span>)
`,wrap:!1}}),de=new C({props:{title:"Model optimization",local:"model-optimization",headingTag:"h2"}}),ce=new C({props:{title:"Quantization using Bitsandbytes for memory efficiency",local:"quantization-using-bitsandbytes-for-memory-efficiency",headingTag:"h3"}}),he=new F({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYXZhTmV4dFZpZGVvRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwTGxhdmFOZXh0VmlkZW9Qcm9jZXNzb3IlMEElMEElMjMlMjBzcGVjaWZ5JTIwaG93JTIwdG8lMjBxdWFudGl6ZSUyMHRoZSUyMG1vZGVsJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwTGxhdmFOZXh0VmlkZW9Gb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmxsYXZhLWhmJTJGTExhVkEtTmVYVC1WaWRlby03Qi1oZiUyMiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlavaNextVideoForConditionalGeneration, LlavaNextVideoProcessor

<span class="hljs-comment"># specify how to quantize the model</span>
quantization_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=torch.float16,
)

model = LlavaNextVideoForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>, quantization_config=quantization_config, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),Me=new C({props:{title:"Flash-Attention 2 to speed-up generation",local:"flash-attention-2-to-speed-up-generation",headingTag:"h3"}}),fe=new F({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZmxhc2gtYXR0biUyMC0tbm8tYnVpbGQtaXNvbGF0aW9u",highlighted:"pip install -U flash-attn --no-build-isolation",wrap:!1}}),Ue=new F({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYXZhTmV4dFZpZGVvRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBJTBBbW9kZWwlMjAlM0QlMjBMbGF2YU5leHRWaWRlb0ZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybGxhdmEtaGYlMkZMTGFWQS1OZVhULVZpZGVvLTdCLWhmJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwJTBBJTIwJTIwJTIwJTIwYXR0bl9pbXBsZW1lbnRhdGlvbiUzRCUyMmZsYXNoX2F0dGVudGlvbl8yJTIyJTJDJTBBKS50bygwKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlavaNextVideoForConditionalGeneration

model = LlavaNextVideoForConditionalGeneration.from_pretrained(
    <span class="hljs-string">&quot;llava-hf/LLaVA-NeXT-Video-7B-hf&quot;</span>, 
    torch_dtype=torch.float16, 
    attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>,
).to(<span class="hljs-number">0</span>)`,wrap:!1}}),ve=new C({props:{title:"LlavaNextVideoConfig",local:"transformers.LlavaNextVideoConfig",headingTag:"h2"}}),_e=new B({props:{name:"class transformers.LlavaNextVideoConfig",anchor:"transformers.LlavaNextVideoConfig",parameters:[{name:"vision_config",val:" = None"},{name:"text_config",val:" = None"},{name:"ignore_index",val:" = -100"},{name:"image_token_index",val:" = 32001"},{name:"projector_hidden_act",val:" = 'gelu'"},{name:"vision_feature_select_strategy",val:" = 'default'"},{name:"vision_feature_layer",val:" = -2"},{name:"image_grid_pinpoints",val:" = None"},{name:"tie_word_embeddings",val:" = False"},{name:"video_token_index",val:" = 32000"},{name:"spatial_pool_mode",val:" = 'average'"},{name:"spatial_pool_stride",val:" = 2"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.LlavaNextVideoConfig.vision_config",description:`<strong>vision_config</strong> (<code>Union[AutoConfig, dict]</code>,  <em>optional</em>, defaults to <code>CLIPVisionConfig</code>) &#x2014;
The config object or dictionary of the vision backbone.`,name:"vision_config"},{anchor:"transformers.LlavaNextVideoConfig.text_config",description:`<strong>text_config</strong> (<code>Union[AutoConfig, dict]</code>, <em>optional</em>, defaults to <code>LlamaConfig</code>) &#x2014;
The config object or dictionary of the text backbone.`,name:"text_config"},{anchor:"transformers.LlavaNextVideoConfig.ignore_index",description:`<strong>ignore_index</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The ignore index for the loss function.`,name:"ignore_index"},{anchor:"transformers.LlavaNextVideoConfig.image_token_index",description:`<strong>image_token_index</strong> (<code>int</code>, <em>optional</em>, defaults to 32001) &#x2014;
The image token index to encode the image prompt.`,name:"image_token_index"},{anchor:"transformers.LlavaNextVideoConfig.projector_hidden_act",description:`<strong>projector_hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The activation function used by the multimodal projector.`,name:"projector_hidden_act"},{anchor:"transformers.LlavaNextVideoConfig.vision_feature_select_strategy",description:`<strong>vision_feature_select_strategy</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;default&quot;</code>) &#x2014;
The feature selection strategy used to select the vision feature from the vision backbone.
Can be one of <code>&quot;default&quot;</code> or <code>&quot;full&quot;</code>. If <code>&quot;default&quot;</code>, the CLS token is removed from the vision features.
If <code>&quot;full&quot;</code>, the full vision features are used.`,name:"vision_feature_select_strategy"},{anchor:"transformers.LlavaNextVideoConfig.vision_feature_layer",description:`<strong>vision_feature_layer</strong> (<code>int</code>, <em>optional</em>, defaults to -2) &#x2014;
The index of the layer to select the vision feature.`,name:"vision_feature_layer"},{anchor:"transformers.LlavaNextVideoConfig.image_grid_pinpoints",description:`<strong>image_grid_pinpoints</strong> (<code>List</code>, <em>optional</em>, defaults to <code>[[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]]</code>) &#x2014;
A list of possible resolutions to use for processing high resolution images. Each item in the list should be a tuple or list
of the form <code>(height, width)</code>.`,name:"image_grid_pinpoints"},{anchor:"transformers.LlavaNextVideoConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the model&#x2019;s input and output word embeddings should be tied.`,name:"tie_word_embeddings"},{anchor:"transformers.LlavaNextVideoConfig.video_token_index",description:`<strong>video_token_index</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
The video token index to encode the image prompt.`,name:"video_token_index"},{anchor:"transformers.LlavaNextVideoConfig.spatial_pool_mode",description:`<strong>spatial_pool_mode</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;average&quot;</code>) &#x2014;
Pooling mode to use for videos. Can be &#x201C;average&#x201D;, &#x201C;max&#x201D; or &#x201C;conv&#x201D;.`,name:"spatial_pool_mode"},{anchor:"transformers.LlavaNextVideoConfig.spatial_pool_stride",description:`<strong>spatial_pool_stride</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
Stride used in the pooling layer for videos.`,name:"spatial_pool_stride"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/configuration_llava_next_video.py#L28"}}),k=new Zt({props:{anchor:"transformers.LlavaNextVideoConfig.example",$$slots:{default:[At]},$$scope:{ctx:G}}}),Je=new C({props:{title:"LlavaNextVideoProcessor",local:"transformers.LlavaNextVideoProcessor",headingTag:"h2"}}),we=new B({props:{name:"class transformers.LlavaNextVideoProcessor",anchor:"transformers.LlavaNextVideoProcessor",parameters:[{name:"video_processor",val:" = None"},{name:"image_processor",val:" = None"},{name:"tokenizer",val:" = None"},{name:"chat_template",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.LlavaNextVideoProcessor.video_processor",description:`<strong>video_processor</strong> (<a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoImageProcessor">LlavaNextVideoImageProcessor</a>, <em>optional</em>) &#x2014;
The video processor is a required input.`,name:"video_processor"},{anchor:"transformers.LlavaNextVideoProcessor.image_processor",description:`<strong>image_processor</strong> (<a href="/docs/transformers/pr_30910/en/model_doc/llava_next#transformers.LlavaNextImageProcessor">LlavaNextImageProcessor</a>, <em>optional</em>) &#x2014;
The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.LlavaNextVideoProcessor.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_30910/en/model_doc/llama#transformers.LlamaTokenizerFast">LlamaTokenizerFast</a>, <em>optional</em>) &#x2014;
The tokenizer is a required input.`,name:"tokenizer"},{anchor:"transformers.LlavaNextVideoProcessor.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Jinja chat template that will be used in tokenizer&#x2019;s <code>apply_chat_template</code>`,name:"chat_template"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/processing_llava_next_video.py#L34"}}),be=new B({props:{name:"batch_decode",anchor:"transformers.LlavaNextVideoProcessor.batch_decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/processing_llava_next_video.py#L141"}}),je=new B({props:{name:"decode",anchor:"transformers.LlavaNextVideoProcessor.decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/processing_llava_next_video.py#L149"}}),Ce=new C({props:{title:"LlavaNextVideoImageProcessor",local:"transformers.LlavaNextVideoImageProcessor",headingTag:"h2"}}),Ie=new B({props:{name:"class transformers.LlavaNextVideoImageProcessor",anchor:"transformers.LlavaNextVideoImageProcessor",parameters:[{name:"do_resize",val:": bool = True"},{name:"size",val:": Dict = None"},{name:"image_grid_pinpoints",val:": List = None"},{name:"resample",val:": Resampling = <Resampling.BICUBIC: 3>"},{name:"do_center_crop",val:": bool = True"},{name:"crop_size",val:": Dict = None"},{name:"do_rescale",val:": bool = True"},{name:"rescale_factor",val:": Union = 0.00392156862745098"},{name:"do_normalize",val:": bool = True"},{name:"image_mean",val:": Union = None"},{name:"image_std",val:": Union = None"},{name:"do_convert_rgb",val:": bool = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.LlavaNextVideoImageProcessor.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to resize the image&#x2019;s (height, width) dimensions to the specified <code>size</code>. Can be overridden by
<code>do_resize</code> in the <code>preprocess</code> method.`,name:"do_resize"},{anchor:"transformers.LlavaNextVideoImageProcessor.size",description:`<strong>size</strong> (<code>Dict[str, int]</code> <em>optional</em>, defaults to <code>{&quot;shortest_edge&quot; -- 224}</code>):
Size of the image after resizing. The shortest edge of the image is resized to size[&#x201C;shortest_edge&#x201D;], with
the longest edge resized to keep the input aspect ratio. Can be overridden by <code>size</code> in the <code>preprocess</code>
method.`,name:"size"},{anchor:"transformers.LlavaNextVideoImageProcessor.image_grid_pinpoints",description:`<strong>image_grid_pinpoints</strong> (<code>List</code> <em>optional</em>, defaults to <code>[[672, 336], [336, 672], [672, 672], [336, 1008], [1008, 336]]</code>) &#x2014;
A list of possible resolutions to use for processing high resolution images. The best resolution is selected
based on the original size of the image. Can be overridden by <code>image_grid_pinpoints</code> in the <code>preprocess</code>
method. Not used for processinf videos.`,name:"image_grid_pinpoints"},{anchor:"transformers.LlavaNextVideoImageProcessor.resample",description:`<strong>resample</strong> (<code>PILImageResampling</code>, <em>optional</em>, defaults to <code>Resampling.BICUBIC</code>) &#x2014;
Resampling filter to use if resizing the image. Can be overridden by <code>resample</code> in the <code>preprocess</code> method.`,name:"resample"},{anchor:"transformers.LlavaNextVideoImageProcessor.do_center_crop",description:`<strong>do_center_crop</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to center crop the image to the specified <code>crop_size</code>. Can be overridden by <code>do_center_crop</code> in the
<code>preprocess</code> method.`,name:"do_center_crop"},{anchor:"transformers.LlavaNextVideoImageProcessor.crop_size",description:`<strong>crop_size</strong> (<code>Dict[str, int]</code> <em>optional</em>, defaults to 224) &#x2014;
Size of the output image after applying <code>center_crop</code>. Can be overridden by <code>crop_size</code> in the <code>preprocess</code>
method.`,name:"crop_size"},{anchor:"transformers.LlavaNextVideoImageProcessor.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to rescale the image by the specified scale <code>rescale_factor</code>. Can be overridden by <code>do_rescale</code> in
the <code>preprocess</code> method.`,name:"do_rescale"},{anchor:"transformers.LlavaNextVideoImageProcessor.rescale_factor",description:`<strong>rescale_factor</strong> (<code>int</code> or <code>float</code>, <em>optional</em>, defaults to <code>1/255</code>) &#x2014;
Scale factor to use if rescaling the image. Can be overridden by <code>rescale_factor</code> in the <code>preprocess</code>
method.`,name:"rescale_factor"},{anchor:"transformers.LlavaNextVideoImageProcessor.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to normalize the image. Can be overridden by <code>do_normalize</code> in the <code>preprocess</code> method.`,name:"do_normalize"},{anchor:"transformers.LlavaNextVideoImageProcessor.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>[0.48145466, 0.4578275, 0.40821073]</code>) &#x2014;
Mean to use if normalizing the image. This is a float or list of floats the length of the number of
channels in the image. Can be overridden by the <code>image_mean</code> parameter in the <code>preprocess</code> method.`,name:"image_mean"},{anchor:"transformers.LlavaNextVideoImageProcessor.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>[0.26862954, 0.26130258, 0.27577711]</code>) &#x2014;
Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
number of channels in the image. Can be overridden by the <code>image_std</code> parameter in the <code>preprocess</code> method.
Can be overridden by the <code>image_std</code> parameter in the <code>preprocess</code> method.`,name:"image_std"},{anchor:"transformers.LlavaNextVideoImageProcessor.do_convert_rgb",description:`<strong>do_convert_rgb</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to convert the image to RGB.`,name:"do_convert_rgb"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/image_processing_llava_next_video.py#L68"}}),xe=new B({props:{name:"preprocess",anchor:"transformers.LlavaNextVideoImageProcessor.preprocess",parameters:[{name:"images",val:": Union"},{name:"do_resize",val:": bool = None"},{name:"size",val:": Dict = None"},{name:"resample",val:": Resampling = None"},{name:"do_center_crop",val:": bool = None"},{name:"crop_size",val:": int = None"},{name:"do_rescale",val:": bool = None"},{name:"rescale_factor",val:": float = None"},{name:"do_normalize",val:": bool = None"},{name:"image_mean",val:": Union = None"},{name:"image_std",val:": Union = None"},{name:"do_convert_rgb",val:": bool = None"},{name:"return_tensors",val:": Union = None"},{name:"data_format",val:": Optional = <ChannelDimension.FIRST: 'channels_first'>"},{name:"input_data_format",val:": Union = None"}],parametersDescription:[{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.images",description:`<strong>images</strong> (<code>VideoInput</code>) &#x2014;
Videos to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
passing in images with pixel values between 0 and 1, set <code>do_rescale=False</code>.`,name:"images"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_resize</code>) &#x2014;
Whether to resize the video.`,name:"do_resize"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.size",description:`<strong>size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>self.size</code>) &#x2014;
Size of the video after resizing. Shortest edge of the video is resized to size[&#x201C;shortest_edge&#x201D;], with
the longest edge resized to keep the input aspect ratio.`,name:"size"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.resample",description:`<strong>resample</strong> (<code>int</code>, <em>optional</em>, defaults to <code>self.resample</code>) &#x2014;
Resampling filter to use if resizing the video. This can be one of the enum <code>PILImageResampling</code>. Only
has an effect if <code>do_resize</code> is set to <code>True</code>.`,name:"resample"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.do_center_crop",description:`<strong>do_center_crop</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_center_crop</code>) &#x2014;
Whether to center crop the video.`,name:"do_center_crop"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.crop_size",description:`<strong>crop_size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>self.crop_size</code>) &#x2014;
Size of the center crop. Only has an effect if <code>do_center_crop</code> is set to <code>True</code>.`,name:"crop_size"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_rescale</code>) &#x2014;
Whether to rescale the video.`,name:"do_rescale"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.rescale_factor",description:`<strong>rescale_factor</strong> (<code>float</code>, <em>optional</em>, defaults to <code>self.rescale_factor</code>) &#x2014;
Rescale factor to rescale the video by if <code>do_rescale</code> is set to <code>True</code>.`,name:"rescale_factor"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_normalize</code>) &#x2014;
Whether to normalize the video.`,name:"do_normalize"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_mean</code>) &#x2014;
Frame mean to use for normalization. Only has an effect if <code>do_normalize</code> is set to <code>True</code>.`,name:"image_mean"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_std</code>) &#x2014;
Frame standard deviation to use for normalization. Only has an effect if <code>do_normalize</code> is set to
<code>True</code>.`,name:"image_std"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.do_convert_rgb",description:`<strong>do_convert_rgb</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_convert_rgb</code>) &#x2014;
Whether to convert the video to RGB.`,name:"do_convert_rgb"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <code>TensorType</code>, <em>optional</em>) &#x2014;
The type of tensors to return. Can be one of:<ul>
<li>Unset: Return a list of <code>np.ndarray</code>.</li>
<li><code>TensorType.TENSORFLOW</code> or <code>&apos;tf&apos;</code>: Return a batch of type <code>tf.Tensor</code>.</li>
<li><code>TensorType.PYTORCH</code> or <code>&apos;pt&apos;</code>: Return a batch of type <code>torch.Tensor</code>.</li>
<li><code>TensorType.NUMPY</code> or <code>&apos;np&apos;</code>: Return a batch of type <code>np.ndarray</code>.</li>
<li><code>TensorType.JAX</code> or <code>&apos;jax&apos;</code>: Return a batch of type <code>jax.numpy.ndarray</code>.</li>
</ul>`,name:"return_tensors"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.data_format",description:`<strong>data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>, defaults to <code>ChannelDimension.FIRST</code>) &#x2014;
The channel dimension format for the output image. Can be one of:<ul>
<li><code>&quot;channels_first&quot;</code> or <code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>&quot;channels_last&quot;</code> or <code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li>Unset: Use the channel dimension format of the input image.</li>
</ul>`,name:"data_format"},{anchor:"transformers.LlavaNextVideoImageProcessor.preprocess.input_data_format",description:`<strong>input_data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>) &#x2014;
The channel dimension format for the input image. If unset, the channel dimension format is inferred
from the input image. Can be one of:<ul>
<li><code>&quot;channels_first&quot;</code> or <code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>&quot;channels_last&quot;</code> or <code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li><code>&quot;none&quot;</code> or <code>ChannelDimension.NONE</code>: image in (height, width) format.</li>
</ul>`,name:"input_data_format"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/image_processing_llava_next_video.py#L304"}}),Ve=new B({props:{name:"resize",anchor:"transformers.LlavaNextVideoImageProcessor.resize",parameters:[{name:"image",val:": ndarray"},{name:"size",val:": Dict"},{name:"resample",val:": Resampling = <Resampling.BICUBIC: 3>"},{name:"data_format",val:": Union = None"},{name:"input_data_format",val:": Union = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.LlavaNextVideoImageProcessor.resize.image",description:`<strong>image</strong> (<code>np.ndarray</code>) &#x2014;
Image to resize.`,name:"image"},{anchor:"transformers.LlavaNextVideoImageProcessor.resize.size",description:`<strong>size</strong> (<code>Dict[str, int]</code>) &#x2014;
Size of the output image.`,name:"size"},{anchor:"transformers.LlavaNextVideoImageProcessor.resize.resample",description:`<strong>resample</strong> (<code>PILImageResampling</code>, <em>optional</em>, defaults to <code>PILImageResampling.BICUBIC</code>) &#x2014;
Resampling filter to use when resiizing the image.`,name:"resample"},{anchor:"transformers.LlavaNextVideoImageProcessor.resize.data_format",description:`<strong>data_format</strong> (<code>str</code> or <code>ChannelDimension</code>, <em>optional</em>) &#x2014;
The channel dimension format of the image. If not provided, it will be the same as the input image.`,name:"data_format"},{anchor:"transformers.LlavaNextVideoImageProcessor.resize.input_data_format",description:`<strong>input_data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>) &#x2014;
The channel dimension format of the input image. If not provided, it will be inferred.`,name:"input_data_format"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/image_processing_llava_next_video.py#L149"}}),Ze=new C({props:{title:"LlavaNextVideoForConditionalGeneration",local:"transformers.LlavaNextVideoForConditionalGeneration",headingTag:"h2"}}),Ne=new B({props:{name:"class transformers.LlavaNextVideoForConditionalGeneration",anchor:"transformers.LlavaNextVideoForConditionalGeneration",parameters:[{name:"config",val:": LlavaNextVideoConfig"}],parametersDescription:[{anchor:"transformers.LlavaNextVideoForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoConfig">LlavaNextVideoConfig</a> or <code>LlavaNextVideoVisionConfig</code>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_30910/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/modeling_llava_next_video.py#L381"}}),Be=new B({props:{name:"forward",anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"pixel_values",val:": FloatTensor = None"},{name:"pixel_values_videos",val:": FloatTensor = None"},{name:"image_sizes",val:": Optional = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"vision_feature_layer",val:": Optional = None"},{name:"vision_feature_select_strategy",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_30910/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape \`(batch_size, num_channels, image_size, image_size)) &#x2014;
The tensors corresponding to the input images. Pixel values can be obtained using
<a href="/docs/transformers/pr_30910/en/model_doc/auto#transformers.AutoImageProcessor">AutoImageProcessor</a>. See <a href="/docs/transformers/pr_30910/en/model_doc/levit#transformers.LevitFeatureExtractor.__call__">LlavaNextVideoImageProcessor.<strong>call</strong>()</a> for details. <a href="/docs/transformers/pr_30910/en/model_doc/llava#transformers.LlavaProcessor">LlavaProcessor</a> uses
<a href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoImageProcessor">LlavaNextVideoImageProcessor</a> for processing images.`,name:"pixel_values"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.image_sizes",description:`<strong>image_sizes</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, 2)</code>, <em>optional</em>) &#x2014;
The sizes of the images in the batch, being (height, width) for each image.`,name:"image_sizes"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_30910/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30910/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>. <a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.vision_feature_layer",description:`<strong>vision_feature_layer</strong> (<code>int</code>, <em>optional</em>, defaults to -2) &#x2014;
The index of the layer to select the vision feature.`,name:"vision_feature_layer"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.vision_feature_select_strategy",description:`<strong>vision_feature_select_strategy</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;default&quot;</code>) &#x2014;
The feature selection strategy used to select the vision feature from the vision backbone.
Can be one of <code>&quot;default&quot;</code> or <code>&quot;full&quot;</code>. If <code>&quot;default&quot;</code>, the CLS token is removed from the vision features.
If <code>&quot;full&quot;</code>, the full vision features are used.`,name:"vision_feature_select_strategy"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/pr_30910/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.</p>
<p>Args &#x2014;
pixel_values_videos (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, image_size, image_size)): The tensors corresponding to the input videos. Pixel values can be obtained using [AutoImageProcessor](/docs/transformers/pr_30910/en/model_doc/auto#transformers.AutoImageProcessor). See </code>LlavaNextVideoVideoProcessor.<strong>call</strong><code>for details. [LlavaProcessor](/docs/transformers/pr_30910/en/model_doc/llava#transformers.LlavaProcessor) uses</code>LlavaNextVideoVideoProcessor<code> for processing videos. labels (</code>torch.LongTensor<code>of shape</code>(batch_size, sequence_length)<code>, *optional*): Labels for computing the masked language modeling loss. Indices should either be in </code>[0, &#x2026;,
config.vocab_size]<code>or -100 (see</code>input_ids<code>docstring). Tokens with indices set to</code>-100<code>are ignored (masked), the loss is only computed for the tokens with labels in</code>[0, &#x2026;, config.vocab_size]\`.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/vr_30910/src/transformers/models/llava_next_video/modeling_llava_next_video.py#L736",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.models.llava_next_video.modeling_llava_next_video.LlavaNextVideoCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_30910/en/model_doc/llava-next-video#transformers.LlavaNextVideoConfig"
>LlavaNextVideoConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
<li>
<p><strong>image_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the image embeddings, <code>(batch_size, num_images, sequence_length, hidden_size)</code>.</p>
<p>image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.models.llava_next_video.modeling_llava_next_video.LlavaNextVideoCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),A=new zt({props:{$$slots:{default:[Rt]},$$scope:{ctx:G}}}),R=new Zt({props:{anchor:"transformers.LlavaNextVideoForConditionalGeneration.forward.example",$$slots:{default:[Xt]},$$scope:{ctx:G}}}),Ge=new Wt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/llava-next-video.md"}}),{c(){g=l("meta"),J=n(),T=l("p"),y=n(),c(U.$$.fragment),i=n(),c(v.$$.fragment),Ye=n(),Q=l("p"),Q.innerHTML=at,Pe=n(),E=l("p"),E.innerHTML=nt,He=n(),S=l("p"),S.textContent=st,De=n(),Y=l("p"),Y.textContent=lt,Ke=n(),P=l("p"),P.textContent=rt,Oe=n(),H=l("ul"),H.innerHTML=it,eo=n(),D=l("p"),D.innerHTML=dt,oo=n(),c(K.$$.fragment),to=n(),O=l("ul"),O.innerHTML=ct,ao=n(),ee=l("p"),ee.innerHTML=mt,no=n(),c(oe.$$.fragment),so=n(),c(te.$$.fragment),lo=n(),c(ae.$$.fragment),ro=n(),ne=l("p"),ne.innerHTML=pt,io=n(),c(se.$$.fragment),co=n(),c(le.$$.fragment),mo=n(),re=l("p"),re.textContent=ht,po=n(),c(ie.$$.fragment),ho=n(),c(de.$$.fragment),Mo=n(),c(ce.$$.fragment),uo=n(),me=l("p"),me.textContent=Mt,go=n(),pe=l("p"),pe.innerHTML=ut,fo=n(),c(he.$$.fragment),yo=n(),c(Me.$$.fragment),To=n(),ue=l("p"),ue.innerHTML=gt,Uo=n(),ge=l("p"),ge.textContent=ft,vo=n(),c(fe.$$.fragment),_o=n(),ye=l("p"),ye.innerHTML=yt,Jo=n(),Te=l("p"),Te.innerHTML=Tt,wo=n(),c(Ue.$$.fragment),bo=n(),c(ve.$$.fragment),jo=n(),x=l("div"),c(_e.$$.fragment),Wo=n(),Le=l("p"),Le.innerHTML=Ut,Ao=n(),c(k.$$.fragment),Co=n(),c(Je.$$.fragment),Io=n(),_=l("div"),c(we.$$.fragment),Ro=n(),ze=l("p"),ze.textContent=vt,Xo=n(),We=l("p"),We.innerHTML=_t,$o=n(),L=l("div"),c(be.$$.fragment),qo=n(),Ae=l("p"),Ae.innerHTML=Jt,Fo=n(),z=l("div"),c(je.$$.fragment),Qo=n(),Re=l("p"),Re.innerHTML=wt,xo=n(),c(Ce.$$.fragment),Vo=n(),w=l("div"),c(Ie.$$.fragment),Eo=n(),Xe=l("p"),Xe.innerHTML=bt,So=n(),$e=l("div"),c(xe.$$.fragment),Yo=n(),W=l("div"),c(Ve.$$.fragment),Po=n(),qe=l("p"),qe.textContent=jt,Zo=n(),c(Ze.$$.fragment),No=n(),b=l("div"),c(Ne.$$.fragment),Ho=n(),Fe=l("p"),Fe.innerHTML=Ct,Do=n(),Qe=l("p"),Qe.innerHTML=It,Ko=n(),I=l("div"),c(Be.$$.fragment),Oo=n(),Ee=l("p"),Ee.innerHTML=xt,et=n(),c(A.$$.fragment),ot=n(),c(R.$$.fragment),Bo=n(),c(Ge.$$.fragment),Go=n(),Se=l("p"),this.h()},l(e){const o=Lt("svelte-u9bgzb",document.head);g=r(o,"META",{name:!0,content:!0}),o.forEach(t),J=s(e),T=r(e,"P",{}),V(T).forEach(t),y=s(e),m(U.$$.fragment,e),i=s(e),m(v.$$.fragment,e),Ye=s(e),Q=r(e,"P",{"data-svelte-h":!0}),f(Q)!=="svelte-fz66h5"&&(Q.innerHTML=at),Pe=s(e),E=r(e,"P",{"data-svelte-h":!0}),f(E)!=="svelte-smxnwa"&&(E.innerHTML=nt),He=s(e),S=r(e,"P",{"data-svelte-h":!0}),f(S)!=="svelte-1fhv1nu"&&(S.textContent=st),De=s(e),Y=r(e,"P",{"data-svelte-h":!0}),f(Y)!=="svelte-1mxjeai"&&(Y.textContent=lt),Ke=s(e),P=r(e,"P",{"data-svelte-h":!0}),f(P)!=="svelte-1p13ikg"&&(P.textContent=rt),Oe=s(e),H=r(e,"UL",{"data-svelte-h":!0}),f(H)!=="svelte-3x3bx4"&&(H.innerHTML=it),eo=s(e),D=r(e,"P",{"data-svelte-h":!0}),f(D)!=="svelte-1ddk3wd"&&(D.innerHTML=dt),oo=s(e),m(K.$$.fragment,e),to=s(e),O=r(e,"UL",{"data-svelte-h":!0}),f(O)!=="svelte-z4narx"&&(O.innerHTML=ct),ao=s(e),ee=r(e,"P",{"data-svelte-h":!0}),f(ee)!=="svelte-1ph49wm"&&(ee.innerHTML=mt),no=s(e),m(oe.$$.fragment,e),so=s(e),m(te.$$.fragment,e),lo=s(e),m(ae.$$.fragment,e),ro=s(e),ne=r(e,"P",{"data-svelte-h":!0}),f(ne)!=="svelte-1pauxcy"&&(ne.innerHTML=pt),io=s(e),m(se.$$.fragment,e),co=s(e),m(le.$$.fragment,e),mo=s(e),re=r(e,"P",{"data-svelte-h":!0}),f(re)!=="svelte-1ngvr6w"&&(re.textContent=ht),po=s(e),m(ie.$$.fragment,e),ho=s(e),m(de.$$.fragment,e),Mo=s(e),m(ce.$$.fragment,e),uo=s(e),me=r(e,"P",{"data-svelte-h":!0}),f(me)!=="svelte-jd6m5t"&&(me.textContent=Mt),go=s(e),pe=r(e,"P",{"data-svelte-h":!0}),f(pe)!=="svelte-v8fva4"&&(pe.innerHTML=ut),fo=s(e),m(he.$$.fragment,e),yo=s(e),m(Me.$$.fragment,e),To=s(e),ue=r(e,"P",{"data-svelte-h":!0}),f(ue)!=="svelte-1bm9h4d"&&(ue.innerHTML=gt),Uo=s(e),ge=r(e,"P",{"data-svelte-h":!0}),f(ge)!=="svelte-13kjllm"&&(ge.textContent=ft),vo=s(e),m(fe.$$.fragment,e),_o=s(e),ye=r(e,"P",{"data-svelte-h":!0}),f(ye)!=="svelte-dzzatn"&&(ye.innerHTML=yt),Jo=s(e),Te=r(e,"P",{"data-svelte-h":!0}),f(Te)!=="svelte-z3ioid"&&(Te.innerHTML=Tt),wo=s(e),m(Ue.$$.fragment,e),bo=s(e),m(ve.$$.fragment,e),jo=s(e),x=r(e,"DIV",{class:!0});var N=V(x);m(_e.$$.fragment,N),Wo=s(N),Le=r(N,"P",{"data-svelte-h":!0}),f(Le)!=="svelte-19dtf75"&&(Le.innerHTML=Ut),Ao=s(N),m(k.$$.fragment,N),N.forEach(t),Co=s(e),m(Je.$$.fragment,e),Io=s(e),_=r(e,"DIV",{class:!0});var j=V(_);m(we.$$.fragment,j),Ro=s(j),ze=r(j,"P",{"data-svelte-h":!0}),f(ze)!=="svelte-ghr7nt"&&(ze.textContent=vt),Xo=s(j),We=r(j,"P",{"data-svelte-h":!0}),f(We)!=="svelte-1f385b9"&&(We.innerHTML=_t),$o=s(j),L=r(j,"DIV",{class:!0});var ke=V(L);m(be.$$.fragment,ke),qo=s(ke),Ae=r(ke,"P",{"data-svelte-h":!0}),f(Ae)!=="svelte-xybne1"&&(Ae.innerHTML=Jt),ke.forEach(t),Fo=s(j),z=r(j,"DIV",{class:!0});var Lo=V(z);m(je.$$.fragment,Lo),Qo=s(Lo),Re=r(Lo,"P",{"data-svelte-h":!0}),f(Re)!=="svelte-4wde9n"&&(Re.innerHTML=wt),Lo.forEach(t),j.forEach(t),xo=s(e),m(Ce.$$.fragment,e),Vo=s(e),w=r(e,"DIV",{class:!0});var X=V(w);m(Ie.$$.fragment,X),Eo=s(X),Xe=r(X,"P",{"data-svelte-h":!0}),f(Xe)!=="svelte-3fj5re"&&(Xe.innerHTML=bt),So=s(X),$e=r(X,"DIV",{class:!0});var Vt=V($e);m(xe.$$.fragment,Vt),Vt.forEach(t),Yo=s(X),W=r(X,"DIV",{class:!0});var zo=V(W);m(Ve.$$.fragment,zo),Po=s(zo),qe=r(zo,"P",{"data-svelte-h":!0}),f(qe)!=="svelte-1qmgo6y"&&(qe.textContent=jt),zo.forEach(t),X.forEach(t),Zo=s(e),m(Ze.$$.fragment,e),No=s(e),b=r(e,"DIV",{class:!0});var $=V(b);m(Ne.$$.fragment,$),Ho=s($),Fe=r($,"P",{"data-svelte-h":!0}),f(Fe)!=="svelte-1uou62u"&&(Fe.innerHTML=Ct),Do=s($),Qe=r($,"P",{"data-svelte-h":!0}),f(Qe)!=="svelte-hswkmf"&&(Qe.innerHTML=It),Ko=s($),I=r($,"DIV",{class:!0});var q=V(I);m(Be.$$.fragment,q),Oo=s(q),Ee=r(q,"P",{"data-svelte-h":!0}),f(Ee)!=="svelte-1ermbu2"&&(Ee.innerHTML=xt),et=s(q),m(A.$$.fragment,q),ot=s(q),m(R.$$.fragment,q),q.forEach(t),$.forEach(t),Bo=s(e),m(Ge.$$.fragment,e),Go=s(e),Se=r(e,"P",{}),V(Se).forEach(t),this.h()},h(){Z(g,"name","hf:doc:metadata"),Z(g,"content",qt),Z(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,o){d(document.head,g),a(e,J,o),a(e,T,o),a(e,y,o),p(U,e,o),a(e,i,o),p(v,e,o),a(e,Ye,o),a(e,Q,o),a(e,Pe,o),a(e,E,o),a(e,He,o),a(e,S,o),a(e,De,o),a(e,Y,o),a(e,Ke,o),a(e,P,o),a(e,Oe,o),a(e,H,o),a(e,eo,o),a(e,D,o),a(e,oo,o),p(K,e,o),a(e,to,o),a(e,O,o),a(e,ao,o),a(e,ee,o),a(e,no,o),p(oe,e,o),a(e,so,o),p(te,e,o),a(e,lo,o),p(ae,e,o),a(e,ro,o),a(e,ne,o),a(e,io,o),p(se,e,o),a(e,co,o),p(le,e,o),a(e,mo,o),a(e,re,o),a(e,po,o),p(ie,e,o),a(e,ho,o),p(de,e,o),a(e,Mo,o),p(ce,e,o),a(e,uo,o),a(e,me,o),a(e,go,o),a(e,pe,o),a(e,fo,o),p(he,e,o),a(e,yo,o),p(Me,e,o),a(e,To,o),a(e,ue,o),a(e,Uo,o),a(e,ge,o),a(e,vo,o),p(fe,e,o),a(e,_o,o),a(e,ye,o),a(e,Jo,o),a(e,Te,o),a(e,wo,o),p(Ue,e,o),a(e,bo,o),p(ve,e,o),a(e,jo,o),a(e,x,o),p(_e,x,null),d(x,Wo),d(x,Le),d(x,Ao),p(k,x,null),a(e,Co,o),p(Je,e,o),a(e,Io,o),a(e,_,o),p(we,_,null),d(_,Ro),d(_,ze),d(_,Xo),d(_,We),d(_,$o),d(_,L),p(be,L,null),d(L,qo),d(L,Ae),d(_,Fo),d(_,z),p(je,z,null),d(z,Qo),d(z,Re),a(e,xo,o),p(Ce,e,o),a(e,Vo,o),a(e,w,o),p(Ie,w,null),d(w,Eo),d(w,Xe),d(w,So),d(w,$e),p(xe,$e,null),d(w,Yo),d(w,W),p(Ve,W,null),d(W,Po),d(W,qe),a(e,Zo,o),p(Ze,e,o),a(e,No,o),a(e,b,o),p(Ne,b,null),d(b,Ho),d(b,Fe),d(b,Do),d(b,Qe),d(b,Ko),d(b,I),p(Be,I,null),d(I,Oo),d(I,Ee),d(I,et),p(A,I,null),d(I,ot),p(R,I,null),a(e,Bo,o),p(Ge,e,o),a(e,Go,o),a(e,Se,o),ko=!0},p(e,[o]){const N={};o&2&&(N.$$scope={dirty:o,ctx:e}),k.$set(N);const j={};o&2&&(j.$$scope={dirty:o,ctx:e}),A.$set(j);const ke={};o&2&&(ke.$$scope={dirty:o,ctx:e}),R.$set(ke)},i(e){ko||(h(U.$$.fragment,e),h(v.$$.fragment,e),h(K.$$.fragment,e),h(oe.$$.fragment,e),h(te.$$.fragment,e),h(ae.$$.fragment,e),h(se.$$.fragment,e),h(le.$$.fragment,e),h(ie.$$.fragment,e),h(de.$$.fragment,e),h(ce.$$.fragment,e),h(he.$$.fragment,e),h(Me.$$.fragment,e),h(fe.$$.fragment,e),h(Ue.$$.fragment,e),h(ve.$$.fragment,e),h(_e.$$.fragment,e),h(k.$$.fragment,e),h(Je.$$.fragment,e),h(we.$$.fragment,e),h(be.$$.fragment,e),h(je.$$.fragment,e),h(Ce.$$.fragment,e),h(Ie.$$.fragment,e),h(xe.$$.fragment,e),h(Ve.$$.fragment,e),h(Ze.$$.fragment,e),h(Ne.$$.fragment,e),h(Be.$$.fragment,e),h(A.$$.fragment,e),h(R.$$.fragment,e),h(Ge.$$.fragment,e),ko=!0)},o(e){M(U.$$.fragment,e),M(v.$$.fragment,e),M(K.$$.fragment,e),M(oe.$$.fragment,e),M(te.$$.fragment,e),M(ae.$$.fragment,e),M(se.$$.fragment,e),M(le.$$.fragment,e),M(ie.$$.fragment,e),M(de.$$.fragment,e),M(ce.$$.fragment,e),M(he.$$.fragment,e),M(Me.$$.fragment,e),M(fe.$$.fragment,e),M(Ue.$$.fragment,e),M(ve.$$.fragment,e),M(_e.$$.fragment,e),M(k.$$.fragment,e),M(Je.$$.fragment,e),M(we.$$.fragment,e),M(be.$$.fragment,e),M(je.$$.fragment,e),M(Ce.$$.fragment,e),M(Ie.$$.fragment,e),M(xe.$$.fragment,e),M(Ve.$$.fragment,e),M(Ze.$$.fragment,e),M(Ne.$$.fragment,e),M(Be.$$.fragment,e),M(A.$$.fragment,e),M(R.$$.fragment,e),M(Ge.$$.fragment,e),ko=!1},d(e){e&&(t(J),t(T),t(y),t(i),t(Ye),t(Q),t(Pe),t(E),t(He),t(S),t(De),t(Y),t(Ke),t(P),t(Oe),t(H),t(eo),t(D),t(oo),t(to),t(O),t(ao),t(ee),t(no),t(so),t(lo),t(ro),t(ne),t(io),t(co),t(mo),t(re),t(po),t(ho),t(Mo),t(uo),t(me),t(go),t(pe),t(fo),t(yo),t(To),t(ue),t(Uo),t(ge),t(vo),t(_o),t(ye),t(Jo),t(Te),t(wo),t(bo),t(jo),t(x),t(Co),t(Io),t(_),t(xo),t(Vo),t(w),t(Zo),t(No),t(b),t(Bo),t(Go),t(Se)),t(g),u(U,e),u(v,e),u(K,e),u(oe,e),u(te,e),u(ae,e),u(se,e),u(le,e),u(ie,e),u(de,e),u(ce,e),u(he,e),u(Me,e),u(fe,e),u(Ue,e),u(ve,e),u(_e),u(k),u(Je,e),u(we),u(be),u(je),u(Ce,e),u(Ie),u(xe),u(Ve),u(Ze,e),u(Ne),u(Be),u(A),u(R),u(Ge,e)}}}const qt='{"title":"LLaVa-NeXT-Video","local":"llava-next-video","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage tips","local":"usage-tips","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[{"title":"Single Media Mode","local":"single-media-mode","sections":[],"depth":3},{"title":"Mixed Media Mode","local":"mixed-media-mode","sections":[],"depth":3}],"depth":2},{"title":"Model optimization","local":"model-optimization","sections":[{"title":"Quantization using Bitsandbytes for memory efficiency","local":"quantization-using-bitsandbytes-for-memory-efficiency","sections":[],"depth":3},{"title":"Flash-Attention 2 to speed-up generation","local":"flash-attention-2-to-speed-up-generation","sections":[],"depth":3}],"depth":2},{"title":"LlavaNextVideoConfig","local":"transformers.LlavaNextVideoConfig","sections":[],"depth":2},{"title":"LlavaNextVideoProcessor","local":"transformers.LlavaNextVideoProcessor","sections":[],"depth":2},{"title":"LlavaNextVideoImageProcessor","local":"transformers.LlavaNextVideoImageProcessor","sections":[],"depth":2},{"title":"LlavaNextVideoForConditionalGeneration","local":"transformers.LlavaNextVideoForConditionalGeneration","sections":[],"depth":2}],"depth":1}';function Ft(G){return Bt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Kt extends Gt{constructor(g){super(),kt(this,g,Ft,$t,Nt,{})}}export{Kt as component};
