import{S as qa,i as Oa,s as Ka,e as n,k as d,w as h,t as l,M as el,c as s,d as o,m as c,a,x as _,h as i,b as p,N as rt,G as t,g as m,y as b,q as y,o as v,B as w,v as tl,L as ol}from"../../../chunks/vendor-hf-doc-builder.js";import{D as O}from"../../../chunks/Docstring-hf-doc-builder.js";import{C as Wt}from"../../../chunks/CodeBlock-hf-doc-builder.js";import{I as Gs}from"../../../chunks/IconCopyLink-hf-doc-builder.js";import{E as nl}from"../../../chunks/ExampleCodeBlock-hf-doc-builder.js";function sl(kt){let M,W,J,T,x;return T=new Wt({props:{code:"JTIzJTIwIXBpcCUyMGluc3RhbGwlMjBvcGVuY3YtcHl0aG9uJTIwdHJhbnNmb3JtZXJzJTIwYWNjZWxlcmF0ZSUwQWZyb20lMjBkaWZmdXNlcnMlMjBpbXBvcnQlMjBTdGFibGVEaWZmdXNpb25YTENvbnRyb2xOZXRQaXBlbGluZSUyQyUyMENvbnRyb2xOZXRNb2RlbCUyQyUyMEF1dG9lbmNvZGVyS0wlMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQWltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBaW1wb3J0JTIwY3YyJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIyYWVyaWFsJTIwdmlldyUyQyUyMGElMjBmdXR1cmlzdGljJTIwcmVzZWFyY2glMjBjb21wbGV4JTIwaW4lMjBhJTIwYnJpZ2h0JTIwZm9nZ3klMjBqdW5nbGUlMkMlMjBoYXJkJTIwbGlnaHRpbmclMjIlMEFuZWdhdGl2ZV9wcm9tcHQlMjAlM0QlMjAlMjJsb3clMjBxdWFsaXR5JTJDJTIwYmFkJTIwcXVhbGl0eSUyQyUyMHNrZXRjaGVzJTIyJTBBJTBBJTIzJTIwZG93bmxvYWQlMjBhbiUyMGltYWdlJTBBaW1hZ2UlMjAlM0QlMjBsb2FkX2ltYWdlKCUwQSUyMCUyMCUyMCUyMCUyMmh0dHBzJTNBJTJGJTJGaGYuY28lMkZkYXRhc2V0cyUyRmhmLWludGVybmFsLXRlc3RpbmclMkZkaWZmdXNlcnMtaW1hZ2VzJTJGcmVzb2x2ZSUyRm1haW4lMkZzZF9jb250cm9sbmV0JTJGaGYtbG9nby5wbmclMjIlMEEpJTBBJTBBJTIzJTIwaW5pdGlhbGl6ZSUyMHRoZSUyMG1vZGVscyUyMGFuZCUyMHBpcGVsaW5lJTBBY29udHJvbG5ldF9jb25kaXRpb25pbmdfc2NhbGUlMjAlM0QlMjAwLjUlMjAlMjAlMjMlMjByZWNvbW1lbmRlZCUyMGZvciUyMGdvb2QlMjBnZW5lcmFsaXphdGlvbiUwQWNvbnRyb2xuZXQlMjAlM0QlMjBDb250cm9sTmV0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpZmZ1c2VycyUyRmNvbnRyb2xuZXQtY2Fubnktc2R4bC0xLjAlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBdmFlJTIwJTNEJTIwQXV0b2VuY29kZXJLTC5mcm9tX3ByZXRyYWluZWQoJTIybWFkZWJ5b2xsaW4lMkZzZHhsLXZhZS1mcDE2LWZpeCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExDb250cm9sTmV0UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi14bC1iYXNlLTEuMCUyMiUyQyUyMGNvbnRyb2xuZXQlM0Rjb250cm9sbmV0JTJDJTIwdmFlJTNEdmFlJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTBBKSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEElMEElMjMlMjBnZXQlMjBjYW5ueSUyMGltYWdlJTBBaW1hZ2UlMjAlM0QlMjBucC5hcnJheShpbWFnZSklMEFpbWFnZSUyMCUzRCUyMGN2Mi5DYW5ueShpbWFnZSUyQyUyMDEwMCUyQyUyMDIwMCklMEFpbWFnZSUyMCUzRCUyMGltYWdlJTVCJTNBJTJDJTIwJTNBJTJDJTIwTm9uZSU1RCUwQWltYWdlJTIwJTNEJTIwbnAuY29uY2F0ZW5hdGUoJTVCaW1hZ2UlMkMlMjBpbWFnZSUyQyUyMGltYWdlJTVEJTJDJTIwYXhpcyUzRDIpJTBBY2FubnlfaW1hZ2UlMjAlM0QlMjBJbWFnZS5mcm9tYXJyYXkoaW1hZ2UpJTBBJTBBJTIzJTIwZ2VuZXJhdGUlMjBpbWFnZSUwQWltYWdlJTIwJTNEJTIwcGlwZSglMEElMjAlMjAlMjAlMjBwcm9tcHQlMkMlMjBjb250cm9sbmV0X2NvbmRpdGlvbmluZ19zY2FsZSUzRGNvbnRyb2xuZXRfY29uZGl0aW9uaW5nX3NjYWxlJTJDJTIwaW1hZ2UlM0RjYW5ueV9pbWFnZSUwQSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># !pip install opencv-python transformers accelerate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> cv2
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;aerial view, a futuristic research complex in a bright foggy jungle, hard lighting&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>negative_prompt = <span class="hljs-string">&quot;low quality, bad quality, sketches&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># download an image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = load_image(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize the models and pipeline</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>controlnet_conditioning_scale = <span class="hljs-number">0.5</span>  <span class="hljs-comment"># recommended for good generalization</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>controlnet = ControlNetModel.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, torch_dtype=torch.float16
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, controlnet=controlnet, vae=vae, torch_dtype=torch.float16
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe.enable_model_cpu_offload()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># get canny image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = np.array(image)
<span class="hljs-meta">&gt;&gt;&gt; </span>image = cv2.Canny(image, <span class="hljs-number">100</span>, <span class="hljs-number">200</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>image = image[:, :, <span class="hljs-literal">None</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>canny_image = Image.fromarray(image)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># generate image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = pipe(
<span class="hljs-meta">... </span>    prompt, controlnet_conditioning_scale=controlnet_conditioning_scale, image=canny_image
<span class="hljs-meta">... </span>).images[<span class="hljs-number">0</span>]`}}),{c(){M=n("p"),W=l("Examples:"),J=d(),h(T.$$.fragment)},l(g){M=s(g,"P",{});var N=a(M);W=i(N,"Examples:"),N.forEach(o),J=c(g),_(T.$$.fragment,g)},m(g,N){m(g,M,N),t(M,W),m(g,J,N),b(T,g,N),x=!0},p:ol,i(g){x||(y(T.$$.fragment,g),x=!0)},o(g){v(T.$$.fragment,g),x=!1},d(g){g&&o(M),g&&o(J),w(T,g)}}}function al(kt){let M,W,J,T,x,g,N,pt,Po,Dt,K,ee,Bo,Vo,Gt,We,Fo,Pt,ke,Yo,Bt,De,dt,$o,Vt,k,Ro,Ge,zo,Qo,Ft,D,Ao,te,Ho,qo,Yt,U,ct,oe,Oo,Ko,mt,ne,en,tn,ft,se,on,nn,ut,ae,sn,$t,G,an,le,ln,rn,Rt,Pe,pn,zt,ie,Ps,Qt,Be,dn,At,E,P,gt,re,cn,ht,mn,Ht,B,fn,_t,un,gn,qt,V,bt,hn,_n,pe,bn,de,yt,yn,vn,Ot,Ve,wn,Kt,Fe,Mn,eo,ce,to,X,me,Ye,Bs,Tn,$e,Jn,Cn,fe,Re,Vs,Un,ze,xn,oo,Qe,Nn,no,ue,so,L,ge,Ae,Fs,In,He,jn,Zn,he,qe,Ys,En,Oe,Xn,ao,I,Ln,Ke,Sn,Wn,et,kn,Dn,lo,_e,io,tt,Gn,ro,be,po,ye,ot,$s,co,S,F,vt,ve,Pn,wt,Bn,mo,f,we,Vn,Mt,Fn,Yn,Me,$n,nt,Rn,zn,Qn,Tt,An,Hn,Te,Je,Jt,qn,On,st,Kn,es,Ce,Ct,ts,os,at,ns,ss,j,Ue,as,Ut,ls,is,Y,rs,$,xe,ps,Ne,ds,xt,cs,ms,fs,R,Ie,us,je,gs,Nt,hs,_s,bs,z,Ze,ys,C,vs,It,ws,Ms,jt,Ts,Js,Zt,Cs,Us,Et,xs,Ns,Is,Q,Ee,js,Xt,Zs,Es,A,Xe,Xs,Lt,Ls,Ss,H,Le,Ws,St,ks,fo;return g=new Gs({}),re=new Gs({}),ce=new Wt({props:{code:"ZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGxvYWRfaW1hZ2UlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMjAlMEFpbXBvcnQlMjBjdjIlMEElMEFjYW5ueV9pbWFnZSUyMCUzRCUyMGxvYWRfaW1hZ2UoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGbGFuZHNjYXBlLnBuZyUyMiUwQSklMEFjYW5ueV9pbWFnZSUyMCUzRCUyMG5wLmFycmF5KGNhbm55X2ltYWdlKSUwQSUwQWxvd190aHJlc2hvbGQlMjAlM0QlMjAxMDAlMEFoaWdoX3RocmVzaG9sZCUyMCUzRCUyMDIwMCUwQSUwQWNhbm55X2ltYWdlJTIwJTNEJTIwY3YyLkNhbm55KGNhbm55X2ltYWdlJTJDJTIwbG93X3RocmVzaG9sZCUyQyUyMGhpZ2hfdGhyZXNob2xkKSUwQSUwQSUyMyUyMHplcm8lMjBvdXQlMjBtaWRkbGUlMjBjb2x1bW5zJTIwb2YlMjBpbWFnZSUyMHdoZXJlJTIwcG9zZSUyMHdpbGwlMjBiZSUyMG92ZXJsYXllZCUwQXplcm9fc3RhcnQlMjAlM0QlMjBjYW5ueV9pbWFnZS5zaGFwZSU1QjElNUQlMjAlMkYlMkYlMjA0JTBBemVyb19lbmQlMjAlM0QlMjB6ZXJvX3N0YXJ0JTIwJTJCJTIwY2FubnlfaW1hZ2Uuc2hhcGUlNUIxJTVEJTIwJTJGJTJGJTIwMiUwQWNhbm55X2ltYWdlJTVCJTNBJTJDJTIwemVyb19zdGFydCUzQXplcm9fZW5kJTVEJTIwJTNEJTIwMCUwQSUwQWNhbm55X2ltYWdlJTIwJTNEJTIwY2FubnlfaW1hZ2UlNUIlM0ElMkMlMjAlM0ElMkMlMjBOb25lJTVEJTBBY2FubnlfaW1hZ2UlMjAlM0QlMjBucC5jb25jYXRlbmF0ZSglNUJjYW5ueV9pbWFnZSUyQyUyMGNhbm55X2ltYWdlJTJDJTIwY2FubnlfaW1hZ2UlNUQlMkMlMjBheGlzJTNEMiklMEFjYW5ueV9pbWFnZSUyMCUzRCUyMEltYWdlLmZyb21hcnJheShjYW5ueV9pbWFnZSkucmVzaXplKCgxMDI0JTJDJTIwMTAyNCkp",highlighted:`<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np 
<span class="hljs-keyword">import</span> cv2

canny_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/landscape.png&quot;</span>
)
canny_image = np.array(canny_image)

low_threshold = <span class="hljs-number">100</span>
high_threshold = <span class="hljs-number">200</span>

canny_image = cv2.Canny(canny_image, low_threshold, high_threshold)

<span class="hljs-comment"># zero out middle columns of image where pose will be overlayed</span>
zero_start = canny_image.shape[<span class="hljs-number">1</span>] // <span class="hljs-number">4</span>
zero_end = zero_start + canny_image.shape[<span class="hljs-number">1</span>] // <span class="hljs-number">2</span>
canny_image[:, zero_start:zero_end] = <span class="hljs-number">0</span>

canny_image = canny_image[:, :, <span class="hljs-literal">None</span>]
canny_image = np.concatenate([canny_image, canny_image, canny_image], axis=<span class="hljs-number">2</span>)
canny_image = Image.fromarray(canny_image).resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))`}}),ue=new Wt({props:{code:"ZnJvbSUyMGNvbnRyb2xuZXRfYXV4JTIwaW1wb3J0JTIwT3BlbnBvc2VEZXRlY3RvciUwQWZyb20lMjBkaWZmdXNlcnMudXRpbHMlMjBpbXBvcnQlMjBsb2FkX2ltYWdlJTBBJTBBb3BlbnBvc2UlMjAlM0QlMjBPcGVucG9zZURldGVjdG9yLmZyb21fcHJldHJhaW5lZCglMjJsbGx5YXN2aWVsJTJGQ29udHJvbE5ldCUyMiklMEElMEFvcGVucG9zZV9pbWFnZSUyMCUzRCUyMGxvYWRfaW1hZ2UoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGcGVyc29uLnBuZyUyMiUwQSklMEFvcGVucG9zZV9pbWFnZSUyMCUzRCUyMG9wZW5wb3NlKG9wZW5wb3NlX2ltYWdlKS5yZXNpemUoKDEwMjQlMkMlMjAxMDI0KSk=",highlighted:`<span class="hljs-keyword">from</span> controlnet_aux <span class="hljs-keyword">import</span> OpenposeDetector
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image

openpose = OpenposeDetector.from_pretrained(<span class="hljs-string">&quot;lllyasviel/ControlNet&quot;</span>)

openpose_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/person.png&quot;</span>
)
openpose_image = openpose(openpose_image).resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))`}}),_e=new Wt({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblhMQ29udHJvbE5ldFBpcGVsaW5lJTJDJTIwQ29udHJvbE5ldE1vZGVsJTJDJTIwQXV0b2VuY29kZXJLTCUyQyUyMFVuaVBDTXVsdGlzdGVwU2NoZWR1bGVyJTBBaW1wb3J0JTIwdG9yY2glMEElMEFjb250cm9sbmV0cyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMENvbnRyb2xOZXRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydGhpYmF1ZCUyRmNvbnRyb2xuZXQtb3BlbnBvc2Utc2R4bC0xLjAlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMjB1c2Vfc2FmZXRlbnNvcnMlM0RUcnVlJTBBJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMENvbnRyb2xOZXRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyZGlmZnVzZXJzJTJGY29udHJvbG5ldC1jYW5ueS1zZHhsLTEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUpJTJDJTBBJTVEJTBBJTBBdmFlJTIwJTNEJTIwQXV0b2VuY29kZXJLTC5mcm9tX3ByZXRyYWluZWQoJTIybWFkZWJ5b2xsaW4lMkZzZHhsLXZhZS1mcDE2LWZpeCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUpJTBBcGlwZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblhMQ29udHJvbE5ldFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24teGwtYmFzZS0xLjAlMjIlMkMlMjBjb250cm9sbmV0JTNEY29udHJvbG5ldHMlMkMlMjB2YWUlM0R2YWUlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMjB1c2Vfc2FmZXRlbnNvcnMlM0RUcnVlJTBBKSUwQXBpcGUuc2NoZWR1bGVyJTIwJTNEJTIwVW5pUENNdWx0aXN0ZXBTY2hlZHVsZXIuZnJvbV9jb25maWcocGlwZS5zY2hlZHVsZXIuY29uZmlnKSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL, UniPCMultistepScheduler
<span class="hljs-keyword">import</span> torch

controlnets = [
    ControlNetModel.from_pretrained(
        <span class="hljs-string">&quot;thibaud/controlnet-openpose-sdxl-1.0&quot;</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>
    ),
    ControlNetModel.from_pretrained(<span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>),
]

vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>)
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, controlnet=controlnets, vae=vae, torch_dtype=torch.float16, use_safetensors=<span class="hljs-literal">True</span>
)
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
pipe.enable_model_cpu_offload()`}}),be=new Wt({props:{code:"cHJvbXB0JTIwJTNEJTIwJTIyYSUyMGdpYW50JTIwc3RhbmRpbmclMjBpbiUyMGElMjBmYW50YXN5JTIwbGFuZHNjYXBlJTJDJTIwYmVzdCUyMHF1YWxpdHklMjIlMEFuZWdhdGl2ZV9wcm9tcHQlMjAlM0QlMjAlMjJtb25vY2hyb21lJTJDJTIwbG93cmVzJTJDJTIwYmFkJTIwYW5hdG9teSUyQyUyMHdvcnN0JTIwcXVhbGl0eSUyQyUyMGxvdyUyMHF1YWxpdHklMjIlMEElMEFnZW5lcmF0b3IlMjAlM0QlMjB0b3JjaC5tYW51YWxfc2VlZCgxKSUwQSUwQWltYWdlcyUyMCUzRCUyMCU1Qm9wZW5wb3NlX2ltYWdlJTJDJTIwY2FubnlfaW1hZ2UlNUQlMEElMEFpbWFnZXMlMjAlM0QlMjBwaXBlKCUwQSUyMCUyMCUyMCUyMHByb21wdCUyQyUwQSUyMCUyMCUyMCUyMGltYWdlJTNEaW1hZ2VzJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDI1JTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTBBJTIwJTIwJTIwJTIwbmVnYXRpdmVfcHJvbXB0JTNEbmVnYXRpdmVfcHJvbXB0JTJDJTBBJTIwJTIwJTIwJTIwbnVtX2ltYWdlc19wZXJfcHJvbXB0JTNEMyUyQyUwQSUyMCUyMCUyMCUyMGNvbnRyb2xuZXRfY29uZGl0aW9uaW5nX3NjYWxlJTNEJTVCMS4wJTJDJTIwMC44JTVEJTJDJTBBKS5pbWFnZXMlNUIwJTVE",highlighted:`prompt = <span class="hljs-string">&quot;a giant standing in a fantasy landscape, best quality&quot;</span>
negative_prompt = <span class="hljs-string">&quot;monochrome, lowres, bad anatomy, worst quality, low quality&quot;</span>

generator = torch.manual_seed(<span class="hljs-number">1</span>)

images = [openpose_image, canny_image]

images = pipe(
    prompt,
    image=images,
    num_inference_steps=<span class="hljs-number">25</span>,
    generator=generator,
    negative_prompt=negative_prompt,
    num_images_per_prompt=<span class="hljs-number">3</span>,
    controlnet_conditioning_scale=[<span class="hljs-number">1.0</span>, <span class="hljs-number">0.8</span>],
).images[<span class="hljs-number">0</span>]`}}),ve=new Gs({}),we=new O({props:{name:"class diffusers.StableDiffusionXLControlNetPipeline",anchor:"diffusers.StableDiffusionXLControlNetPipeline",parameters:[{name:"vae",val:": AutoencoderKL"},{name:"text_encoder",val:": CLIPTextModel"},{name:"text_encoder_2",val:": CLIPTextModelWithProjection"},{name:"tokenizer",val:": CLIPTokenizer"},{name:"tokenizer_2",val:": CLIPTokenizer"},{name:"unet",val:": UNet2DConditionModel"},{name:"controlnet",val:": typing.Union[diffusers.models.controlnet.ControlNetModel, typing.List[diffusers.models.controlnet.ControlNetModel], typing.Tuple[diffusers.models.controlnet.ControlNetModel], diffusers.pipelines.controlnet.multicontrolnet.MultiControlNetModel]"},{name:"scheduler",val:": KarrasDiffusionSchedulers"},{name:"force_zeros_for_empty_prompt",val:": bool = True"},{name:"add_watermarker",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.vae",description:`<strong>vae</strong> (<a href="/docs/diffusers/v0.20.0/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a>) &#x2014;
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.`,name:"vae"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.text_encoder",description:`<strong>text_encoder</strong> (<code>CLIPTextModel</code>) &#x2014;
Frozen text-encoder. Stable Diffusion uses the text portion of
<a href="https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel" rel="nofollow">CLIP</a>, specifically
the <a href="https://huggingface.co/openai/clip-vit-large-patch14" rel="nofollow">clip-vit-large-patch14</a> variant.`,name:"text_encoder"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.text_encoder_2",description:`<strong>text_encoder_2</strong> (<code> CLIPTextModelWithProjection</code>) &#x2014;
Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
<a href="https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection" rel="nofollow">CLIP</a>,
specifically the
<a href="https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k" rel="nofollow">laion/CLIP-ViT-bigG-14-laion2B-39B-b160k</a>
variant.`,name:"text_encoder_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.tokenizer",description:`<strong>tokenizer</strong> (<code>CLIPTokenizer</code>) &#x2014;
Tokenizer of class
<a href="https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer" rel="nofollow">CLIPTokenizer</a>.`,name:"tokenizer"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.tokenizer_2",description:`<strong>tokenizer_2</strong> (<code>CLIPTokenizer</code>) &#x2014;
Second Tokenizer of class
<a href="https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer" rel="nofollow">CLIPTokenizer</a>.`,name:"tokenizer_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.unet",description:'<strong>unet</strong> (<a href="/docs/diffusers/v0.20.0/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel">UNet2DConditionModel</a>) &#x2014; Conditional U-Net architecture to denoise the encoded image latents.',name:"unet"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.controlnet",description:`<strong>controlnet</strong> (<a href="/docs/diffusers/v0.20.0/en/api/models/controlnet#diffusers.ControlNetModel">ControlNetModel</a> or <code>List[ControlNetModel]</code>) &#x2014;
Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets
as a list, the outputs from each ControlNet are added together to create one combined additional
conditioning.`,name:"controlnet"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.scheduler",description:`<strong>scheduler</strong> (<a href="/docs/diffusers/v0.20.0/en/api/schedulers/overview#diffusers.SchedulerMixin">SchedulerMixin</a>) &#x2014;
A scheduler to be used in combination with <code>unet</code> to denoise the encoded image latents. Can be one of
<a href="/docs/diffusers/v0.20.0/en/api/schedulers/ddim#diffusers.DDIMScheduler">DDIMScheduler</a>, <a href="/docs/diffusers/v0.20.0/en/api/schedulers/lms_discrete#diffusers.LMSDiscreteScheduler">LMSDiscreteScheduler</a>, or <a href="/docs/diffusers/v0.20.0/en/api/schedulers/pndm#diffusers.PNDMScheduler">PNDMScheduler</a>.`,name:"scheduler"}],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L104"}}),Ue=new O({props:{name:"__call__",anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__",parameters:[{name:"prompt",val:": typing.Union[str, typing.List[str]] = None"},{name:"prompt_2",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"image",val:": typing.Union[torch.FloatTensor, PIL.Image.Image, numpy.ndarray, typing.List[torch.FloatTensor], typing.List[PIL.Image.Image], typing.List[numpy.ndarray]] = None"},{name:"height",val:": typing.Optional[int] = None"},{name:"width",val:": typing.Optional[int] = None"},{name:"num_inference_steps",val:": int = 50"},{name:"guidance_scale",val:": float = 5.0"},{name:"negative_prompt",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"negative_prompt_2",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"num_images_per_prompt",val:": typing.Optional[int] = 1"},{name:"eta",val:": float = 0.0"},{name:"generator",val:": typing.Union[torch._C.Generator, typing.List[torch._C.Generator], NoneType] = None"},{name:"latents",val:": typing.Optional[torch.FloatTensor] = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_type",val:": typing.Optional[str] = 'pil'"},{name:"return_dict",val:": bool = True"},{name:"callback",val:": typing.Union[typing.Callable[[int, int, torch.FloatTensor], NoneType], NoneType] = None"},{name:"callback_steps",val:": int = 1"},{name:"cross_attention_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"controlnet_conditioning_scale",val:": typing.Union[float, typing.List[float]] = 1.0"},{name:"guess_mode",val:": bool = False"},{name:"control_guidance_start",val:": typing.Union[float, typing.List[float]] = 0.0"},{name:"control_guidance_end",val:": typing.Union[float, typing.List[float]] = 1.0"},{name:"original_size",val:": typing.Tuple[int, int] = None"},{name:"crops_coords_top_left",val:": typing.Tuple[int, int] = (0, 0)"},{name:"target_size",val:": typing.Tuple[int, int] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to guide the image generation. If not defined, one has to pass <code>prompt_embeds</code>.
instead.`,name:"prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt_2",description:`<strong>prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to be sent to the <code>tokenizer_2</code> and <code>text_encoder_2</code>. If not defined, <code>prompt</code> is
used in both text-encoders`,name:"prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.image",description:`<strong>image</strong> (<code>torch.FloatTensor</code>, <code>PIL.Image.Image</code>, <code>np.ndarray</code>, <code>List[torch.FloatTensor]</code>, <code>List[PIL.Image.Image]</code>, <code>List[np.ndarray]</code>, &#x2014;
<code>List[List[torch.FloatTensor]]</code>, <code>List[List[np.ndarray]]</code> or <code>List[List[PIL.Image.Image]]</code>):
The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If
the type is specified as <code>Torch.FloatTensor</code>, it is passed to ControlNet as is. <code>PIL.Image.Image</code> can
also be accepted as an image. The dimensions of the output image defaults to <code>image</code>&#x2019;s dimensions. If
height and/or width are passed, <code>image</code> is resized according to them. If multiple ControlNets are
specified in init, images must be passed as a list such that each element of the list can be correctly
batched for input to a single controlnet.`,name:"image"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.height",description:`<strong>height</strong> (<code>int</code>, <em>optional</em>, defaults to self.unet.config.sample_size * self.vae_scale_factor) &#x2014;
The height in pixels of the generated image.`,name:"height"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.width",description:`<strong>width</strong> (<code>int</code>, <em>optional</em>, defaults to self.unet.config.sample_size * self.vae_scale_factor) &#x2014;
The width in pixels of the generated image.`,name:"width"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.num_inference_steps",description:`<strong>num_inference_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 50) &#x2014;
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
expense of slower inference.`,name:"num_inference_steps"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.guidance_scale",description:`<strong>guidance_scale</strong> (<code>float</code>, <em>optional</em>, defaults to 7.5) &#x2014;
Guidance scale as defined in <a href="https://arxiv.org/abs/2207.12598" rel="nofollow">Classifier-Free Diffusion Guidance</a>.
<code>guidance_scale</code> is defined as <code>w</code> of equation 2. of <a href="https://arxiv.org/pdf/2205.11487.pdf" rel="nofollow">Imagen
Paper</a>. Guidance scale is enabled by setting <code>guidance_scale &gt; 1</code>. Higher guidance scale encourages to generate images that are closely linked to the text <code>prompt</code>,
usually at the expense of lower image quality.`,name:"guidance_scale"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation. If not defined, one has to pass
<code>negative_prompt_embeds</code> instead. Ignored when not using guidance (i.e., ignored if <code>guidance_scale</code> is
less than <code>1</code>).`,name:"negative_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt_2",description:`<strong>negative_prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation to be sent to <code>tokenizer_2</code> and
<code>text_encoder_2</code>. If not defined, <code>negative_prompt</code> is used in both text-encoders`,name:"negative_prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.num_images_per_prompt",description:`<strong>num_images_per_prompt</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of images to generate per prompt.`,name:"num_images_per_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.eta",description:`<strong>eta</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Corresponds to parameter eta (&#x3B7;) in the DDIM paper: <a href="https://arxiv.org/abs/2010.02502" rel="nofollow">https://arxiv.org/abs/2010.02502</a>. Only applies to
<a href="/docs/diffusers/v0.20.0/en/api/schedulers/ddim#diffusers.DDIMScheduler">schedulers.DDIMScheduler</a>, will be ignored for others.`,name:"eta"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.generator",description:`<strong>generator</strong> (<code>torch.Generator</code> or <code>List[torch.Generator]</code>, <em>optional</em>) &#x2014;
One or a list of <a href="https://pytorch.org/docs/stable/generated/torch.Generator.html" rel="nofollow">torch generator(s)</a>
to make generation deterministic.`,name:"generator"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.latents",description:`<strong>latents</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
tensor will ge generated by sampling using the supplied random <code>generator</code>.`,name:"latents"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting. If not
provided, text embeddings will be generated from <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, negative_prompt_embeds will be generated from <code>negative_prompt</code> input
argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.pooled_prompt_embeds",description:`<strong>pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting.
If not provided, pooled text embeddings will be generated from <code>prompt</code> input argument.`,name:"pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_pooled_prompt_embeds",description:`<strong>negative_pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, pooled negative_prompt_embeds will be generated from <code>negative_prompt</code>
input argument.`,name:"negative_pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.output_type",description:`<strong>output_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pil&quot;</code>) &#x2014;
The output format of the generate image. Choose between
<a href="https://pillow.readthedocs.io/en/stable/" rel="nofollow">PIL</a>: <code>PIL.Image.Image</code> or <code>np.array</code>.`,name:"output_type"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to return a <a href="/docs/diffusers/v0.20.0/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput">StableDiffusionPipelineOutput</a> instead of a
plain tuple.`,name:"return_dict"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.callback",description:`<strong>callback</strong> (<code>Callable</code>, <em>optional</em>) &#x2014;
A function that will be called every <code>callback_steps</code> steps during inference. The function will be
called with the following arguments: <code>callback(step: int, timestep: int, latents: torch.FloatTensor)</code>.`,name:"callback"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.callback_steps",description:`<strong>callback_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The frequency at which the <code>callback</code> function will be called. If not specified, the callback will be
called at every step.`,name:"callback_steps"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.cross_attention_kwargs",description:`<strong>cross_attention_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
A kwargs dictionary that if specified is passed along to the <code>AttentionProcessor</code> as defined under
<code>self.processor</code> in
<a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py" rel="nofollow">diffusers.models.attention_processor</a>.`,name:"cross_attention_kwargs"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.controlnet_conditioning_scale",description:`<strong>controlnet_conditioning_scale</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The outputs of the controlnet are multiplied by <code>controlnet_conditioning_scale</code> before they are added
to the residual in the original unet. If multiple ControlNets are specified in init, you can set the
corresponding scale as a list.`,name:"controlnet_conditioning_scale"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.guess_mode",description:`<strong>guess_mode</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
In this mode, the ControlNet encoder will try best to recognize the content of the input image even if
you remove all prompts. The <code>guidance_scale</code> between 3.0 and 5.0 is recommended.`,name:"guess_mode"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.control_guidance_start",description:`<strong>control_guidance_start</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The percentage of total steps at which the controlnet starts applying.`,name:"control_guidance_start"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.control_guidance_end",description:`<strong>control_guidance_end</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The percentage of total steps at which the controlnet stops applying.`,name:"control_guidance_end"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.original_size",description:`<strong>original_size</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (1024, 1024)) &#x2014;
If <code>original_size</code> is not the same as <code>target_size</code> the image will appear to be down- or upsampled.
<code>original_size</code> defaults to <code>(width, height)</code> if not specified. Part of SDXL&#x2019;s micro-conditioning as
explained in section 2.2 of
<a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"original_size"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.crops_coords_top_left",description:`<strong>crops_coords_top_left</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (0, 0)) &#x2014;
<code>crops_coords_top_left</code> can be used to generate an image that appears to be &#x201C;cropped&#x201D; from the position
<code>crops_coords_top_left</code> downwards. Favorable, well-centered images are usually achieved by setting
<code>crops_coords_top_left</code> to (0, 0). Part of SDXL&#x2019;s micro-conditioning as explained in section 2.2 of
<a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"crops_coords_top_left"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.target_size",description:`<strong>target_size</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (1024, 1024)) &#x2014;
For most cases, <code>target_size</code> should be set to the desired height and width of the generated image. If
not specified it will default to <code>(width, height)</code>. Part of SDXL&#x2019;s micro-conditioning as explained in
section 2.2 of <a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"target_size"}],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L751",returnDescription:`
<p><a
  href="/docs/diffusers/v0.20.0/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> if <code>return_dict</code> is True, otherwise a <code>tuple</code>
containing the output images.</p>
`,returnType:`
<p><a
  href="/docs/diffusers/v0.20.0/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> or <code>tuple</code></p>
`}}),Y=new nl({props:{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.example",$$slots:{default:[sl]},$$scope:{ctx:kt}}}),xe=new O({props:{name:"disable_vae_slicing",anchor:"diffusers.StableDiffusionXLControlNetPipeline.disable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L195"}}),Ie=new O({props:{name:"disable_vae_tiling",anchor:"diffusers.StableDiffusionXLControlNetPipeline.disable_vae_tiling",parameters:[],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L212"}}),Ze=new O({props:{name:"enable_model_cpu_offload",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_model_cpu_offload",parameters:[{name:"gpu_id",val:" = 0"}],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L219"}}),Ee=new O({props:{name:"enable_vae_slicing",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L187"}}),Xe=new O({props:{name:"enable_vae_tiling",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_vae_tiling",parameters:[],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L203"}}),Le=new O({props:{name:"encode_prompt",anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt",parameters:[{name:"prompt",val:": str"},{name:"prompt_2",val:": typing.Optional[str] = None"},{name:"device",val:": typing.Optional[torch.device] = None"},{name:"num_images_per_prompt",val:": int = 1"},{name:"do_classifier_free_guidance",val:": bool = True"},{name:"negative_prompt",val:": typing.Optional[str] = None"},{name:"negative_prompt_2",val:": typing.Optional[str] = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"lora_scale",val:": typing.Optional[float] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
prompt to be encoded`,name:"prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt_2",description:`<strong>prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to be sent to the <code>tokenizer_2</code> and <code>text_encoder_2</code>. If not defined, <code>prompt</code> is
used in both text-encoders
device &#x2014; (<code>torch.device</code>):
torch device`,name:"prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.num_images_per_prompt",description:`<strong>num_images_per_prompt</strong> (<code>int</code>) &#x2014;
number of images that should be generated per prompt`,name:"num_images_per_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.do_classifier_free_guidance",description:`<strong>do_classifier_free_guidance</strong> (<code>bool</code>) &#x2014;
whether to use classifier free guidance or not`,name:"do_classifier_free_guidance"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation. If not defined, one has to pass
<code>negative_prompt_embeds</code> instead. Ignored when not using guidance (i.e., ignored if <code>guidance_scale</code> is
less than <code>1</code>).`,name:"negative_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt_2",description:`<strong>negative_prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation to be sent to <code>tokenizer_2</code> and
<code>text_encoder_2</code>. If not defined, <code>negative_prompt</code> is used in both text-encoders`,name:"negative_prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting. If not
provided, text embeddings will be generated from <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, negative_prompt_embeds will be generated from <code>negative_prompt</code> input
argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.pooled_prompt_embeds",description:`<strong>pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting.
If not provided, pooled text embeddings will be generated from <code>prompt</code> input argument.`,name:"pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_pooled_prompt_embeds",description:`<strong>negative_pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, pooled negative_prompt_embeds will be generated from <code>negative_prompt</code>
input argument.`,name:"negative_pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.lora_scale",description:`<strong>lora_scale</strong> (<code>float</code>, <em>optional</em>) &#x2014;
A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.`,name:"lora_scale"}],source:"https://github.com/huggingface/diffusers/blob/v0.20.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L252"}}),{c(){M=n("meta"),W=d(),J=n("h1"),T=n("a"),x=n("span"),h(g.$$.fragment),N=d(),pt=n("span"),Po=l("ControlNet with Stable Diffusion XL"),Dt=d(),K=n("p"),ee=n("a"),Bo=l("Adding Conditional Control to Text-to-Image Diffusion Models"),Vo=l(" by Lvmin Zhang and Maneesh Agrawala."),Gt=d(),We=n("p"),Fo=l("Using a pretrained model, we can provide control images (for example, a depth map) to control Stable Diffusion text-to-image generation so that it follows the structure of the depth image and fills in the details."),Pt=d(),ke=n("p"),Yo=l("The abstract from the paper is:"),Bt=d(),De=n("p"),dt=n("em"),$o=l("We present a neural network structure, ControlNet, to control pretrained large diffusion models to support additional input conditions. The ControlNet learns task-specific conditions in an end-to-end way, and the learning is robust even when the training dataset is small (< 50k). Moreover, training a ControlNet is as fast as fine-tuning a diffusion model, and the model can be trained on a personal devices. Alternatively, if powerful computation clusters are available, the model can scale to large amounts (millions to billions) of data. We report that large diffusion models like Stable Diffusion can be augmented with ControlNets to enable conditional inputs like edge maps, segmentation maps, keypoints, etc. This may enrich the methods to control large diffusion models and further facilitate related applications."),Vt=d(),k=n("p"),Ro=l("We provide support using ControlNets with "),Ge=n("a"),zo=l("Stable Diffusion XL"),Qo=l(" (SDXL)."),Ft=d(),D=n("p"),Ao=l("You can find numerous SDXL ControlNet checkpoints from "),te=n("a"),Ho=l("this link"),qo=l(". There are some smaller ControlNet checkpoints too:"),Yt=d(),U=n("ul"),ct=n("li"),oe=n("a"),Oo=l("controlnet-canny-sdxl-1.0-small"),Ko=d(),mt=n("li"),ne=n("a"),en=l("controlnet-canny-sdxl-1.0-mid"),tn=d(),ft=n("li"),se=n("a"),on=l("controlnet-depth-sdxl-1.0-small"),nn=d(),ut=n("li"),ae=n("a"),sn=l("controlnet-depth-sdxl-1.0-mid"),$t=d(),G=n("p"),an=l("We also encourage you to train custom ControlNets; we provide a "),le=n("a"),ln=l("training script"),rn=l(" for this."),Rt=d(),Pe=n("p"),pn=l("You can find some results below:"),zt=d(),ie=n("img"),Qt=d(),Be=n("p"),dn=l("\u{1F6A8} At the time of this writing, many of these SDXL ControlNet checkpoints are experimental and there is a lot of room for improvement. We encourage our users to provide feedback. \u{1F6A8}"),At=d(),E=n("h2"),P=n("a"),gt=n("span"),h(re.$$.fragment),cn=d(),ht=n("span"),mn=l("MultiControlNet"),Ht=d(),B=n("p"),fn=l("You can compose multiple ControlNet conditionings from different image inputs to create a "),_t=n("em"),un=l("MultiControlNet"),gn=l(". To get better results, it is often helpful to:"),qt=d(),V=n("ol"),bt=n("li"),hn=l("mask conditionings such that they don\u2019t overlap (for example, mask the area of a canny image where the pose conditioning is located)"),_n=d(),pe=n("li"),bn=l("experiment with the "),de=n("a"),yt=n("code"),yn=l("controlnet_conditioning_scale"),vn=l(" parameter to determine how much weight to assign to each conditioning input"),Ot=d(),Ve=n("p"),wn=l("In this example, you\u2019ll combine a canny image and a human pose estimation image to generate a new image."),Kt=d(),Fe=n("p"),Mn=l("Prepare the canny image conditioning:"),eo=d(),h(ce.$$.fragment),to=d(),X=n("div"),me=n("div"),Ye=n("img"),Tn=d(),$e=n("figcaption"),Jn=l("original image"),Cn=d(),fe=n("div"),Re=n("img"),Un=d(),ze=n("figcaption"),xn=l("canny image"),oo=d(),Qe=n("p"),Nn=l("Prepare the human pose estimation conditioning:"),no=d(),h(ue.$$.fragment),so=d(),L=n("div"),ge=n("div"),Ae=n("img"),In=d(),He=n("figcaption"),jn=l("original image"),Zn=d(),he=n("div"),qe=n("img"),En=d(),Oe=n("figcaption"),Xn=l("human pose image"),ao=d(),I=n("p"),Ln=l("Load a list of ControlNet models that correspond to each conditioning, and pass them to the "),Ke=n("a"),Sn=l("StableDiffusionXLControlNetPipeline"),Wn=l(". Use the faster "),et=n("a"),kn=l("UniPCMultistepScheduler"),Dn=l(" and nable model offloading to reduce memory usage."),lo=d(),h(_e.$$.fragment),io=d(),tt=n("p"),Gn=l("Now you can pass your prompt (an optional negative prompt if you\u2019re using one), canny image, and pose image to the pipeline:"),ro=d(),h(be.$$.fragment),po=d(),ye=n("div"),ot=n("img"),co=d(),S=n("h2"),F=n("a"),vt=n("span"),h(ve.$$.fragment),Pn=d(),wt=n("span"),Bn=l("StableDiffusionXLControlNetPipeline"),mo=d(),f=n("div"),h(we.$$.fragment),Vn=d(),Mt=n("p"),Fn=l("Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance."),Yn=d(),Me=n("p"),$n=l("This model inherits from "),nt=n("a"),Rn=l("DiffusionPipeline"),zn=l(`. Check the superclass documentation for the generic methods the
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)`),Qn=d(),Tt=n("p"),An=l("In addition the pipeline inherits the following loading methods:"),Hn=d(),Te=n("ul"),Je=n("li"),Jt=n("em"),qn=l("Textual-Inversion"),On=l(": "),st=n("a"),Kn=l("loaders.TextualInversionLoaderMixin.load_textual_inversion()"),es=d(),Ce=n("li"),Ct=n("em"),ts=l("LoRA"),os=l(": "),at=n("a"),ns=l("loaders.LoraLoaderMixin.load_lora_weights()"),ss=d(),j=n("div"),h(Ue.$$.fragment),as=d(),Ut=n("p"),ls=l("Function invoked when calling the pipeline for generation."),is=d(),h(Y.$$.fragment),rs=d(),$=n("div"),h(xe.$$.fragment),ps=d(),Ne=n("p"),ds=l("Disable sliced VAE decoding. If "),xt=n("code"),cs=l("enable_vae_slicing"),ms=l(` was previously enabled, this method will go back to
computing decoding in one step.`),fs=d(),R=n("div"),h(Ie.$$.fragment),us=d(),je=n("p"),gs=l("Disable tiled VAE decoding. If "),Nt=n("code"),hs=l("enable_vae_tiling"),_s=l(` was previously enabled, this method will go back to
computing decoding in one step.`),bs=d(),z=n("div"),h(Ze.$$.fragment),ys=d(),C=n("p"),vs=l(`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),It=n("code"),ws=l("enable_sequential_cpu_offload"),Ms=l(", this method moves one whole model at a time to the GPU when its "),jt=n("code"),Ts=l("forward"),Js=l(`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Zt=n("code"),Cs=l("enable_sequential_cpu_offload"),Us=l(", but performance is much better due to the iterative execution of the "),Et=n("code"),xs=l("unet"),Ns=l("."),Is=d(),Q=n("div"),h(Ee.$$.fragment),js=d(),Xt=n("p"),Zs=l(`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),Es=d(),A=n("div"),h(Xe.$$.fragment),Xs=d(),Lt=n("p"),Ls=l(`Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
processing larger images.`),Ss=d(),H=n("div"),h(Le.$$.fragment),Ws=d(),St=n("p"),ks=l("Encodes the prompt into text encoder hidden states."),this.h()},l(e){const r=el('[data-svelte="svelte-1phssyn"]',document.head);M=s(r,"META",{name:!0,content:!0}),r.forEach(o),W=c(e),J=s(e,"H1",{class:!0});var Se=a(J);T=s(Se,"A",{id:!0,class:!0,href:!0});var Rs=a(T);x=s(Rs,"SPAN",{});var zs=a(x);_(g.$$.fragment,zs),zs.forEach(o),Rs.forEach(o),N=c(Se),pt=s(Se,"SPAN",{});var Qs=a(pt);Po=i(Qs,"ControlNet with Stable Diffusion XL"),Qs.forEach(o),Se.forEach(o),Dt=c(e),K=s(e,"P",{});var Ds=a(K);ee=s(Ds,"A",{href:!0,rel:!0});var As=a(ee);Bo=i(As,"Adding Conditional Control to Text-to-Image Diffusion Models"),As.forEach(o),Vo=i(Ds," by Lvmin Zhang and Maneesh Agrawala."),Ds.forEach(o),Gt=c(e),We=s(e,"P",{});var Hs=a(We);Fo=i(Hs,"Using a pretrained model, we can provide control images (for example, a depth map) to control Stable Diffusion text-to-image generation so that it follows the structure of the depth image and fills in the details."),Hs.forEach(o),Pt=c(e),ke=s(e,"P",{});var qs=a(ke);Yo=i(qs,"The abstract from the paper is:"),qs.forEach(o),Bt=c(e),De=s(e,"P",{});var Os=a(De);dt=s(Os,"EM",{});var Ks=a(dt);$o=i(Ks,"We present a neural network structure, ControlNet, to control pretrained large diffusion models to support additional input conditions. The ControlNet learns task-specific conditions in an end-to-end way, and the learning is robust even when the training dataset is small (< 50k). Moreover, training a ControlNet is as fast as fine-tuning a diffusion model, and the model can be trained on a personal devices. Alternatively, if powerful computation clusters are available, the model can scale to large amounts (millions to billions) of data. We report that large diffusion models like Stable Diffusion can be augmented with ControlNets to enable conditional inputs like edge maps, segmentation maps, keypoints, etc. This may enrich the methods to control large diffusion models and further facilitate related applications."),Ks.forEach(o),Os.forEach(o),Vt=c(e),k=s(e,"P",{});var uo=a(k);Ro=i(uo,"We provide support using ControlNets with "),Ge=s(uo,"A",{href:!0});var ea=a(Ge);zo=i(ea,"Stable Diffusion XL"),ea.forEach(o),Qo=i(uo," (SDXL)."),uo.forEach(o),Ft=c(e),D=s(e,"P",{});var go=a(D);Ao=i(go,"You can find numerous SDXL ControlNet checkpoints from "),te=s(go,"A",{href:!0,rel:!0});var ta=a(te);Ho=i(ta,"this link"),ta.forEach(o),qo=i(go,". There are some smaller ControlNet checkpoints too:"),go.forEach(o),Yt=c(e),U=s(e,"UL",{});var q=a(U);ct=s(q,"LI",{});var oa=a(ct);oe=s(oa,"A",{href:!0,rel:!0});var na=a(oe);Oo=i(na,"controlnet-canny-sdxl-1.0-small"),na.forEach(o),oa.forEach(o),Ko=c(q),mt=s(q,"LI",{});var sa=a(mt);ne=s(sa,"A",{href:!0,rel:!0});var aa=a(ne);en=i(aa,"controlnet-canny-sdxl-1.0-mid"),aa.forEach(o),sa.forEach(o),tn=c(q),ft=s(q,"LI",{});var la=a(ft);se=s(la,"A",{href:!0,rel:!0});var ia=a(se);on=i(ia,"controlnet-depth-sdxl-1.0-small"),ia.forEach(o),la.forEach(o),nn=c(q),ut=s(q,"LI",{});var ra=a(ut);ae=s(ra,"A",{href:!0,rel:!0});var pa=a(ae);sn=i(pa,"controlnet-depth-sdxl-1.0-mid"),pa.forEach(o),ra.forEach(o),q.forEach(o),$t=c(e),G=s(e,"P",{});var ho=a(G);an=i(ho,"We also encourage you to train custom ControlNets; we provide a "),le=s(ho,"A",{href:!0,rel:!0});var da=a(le);ln=i(da,"training script"),da.forEach(o),rn=i(ho," for this."),ho.forEach(o),Rt=c(e),Pe=s(e,"P",{});var ca=a(Pe);pn=i(ca,"You can find some results below:"),ca.forEach(o),zt=c(e),ie=s(e,"IMG",{src:!0,width:!0}),Qt=c(e),Be=s(e,"P",{});var ma=a(Be);dn=i(ma,"\u{1F6A8} At the time of this writing, many of these SDXL ControlNet checkpoints are experimental and there is a lot of room for improvement. We encourage our users to provide feedback. \u{1F6A8}"),ma.forEach(o),At=c(e),E=s(e,"H2",{class:!0});var _o=a(E);P=s(_o,"A",{id:!0,class:!0,href:!0});var fa=a(P);gt=s(fa,"SPAN",{});var ua=a(gt);_(re.$$.fragment,ua),ua.forEach(o),fa.forEach(o),cn=c(_o),ht=s(_o,"SPAN",{});var ga=a(ht);mn=i(ga,"MultiControlNet"),ga.forEach(o),_o.forEach(o),Ht=c(e),B=s(e,"P",{});var bo=a(B);fn=i(bo,"You can compose multiple ControlNet conditionings from different image inputs to create a "),_t=s(bo,"EM",{});var ha=a(_t);un=i(ha,"MultiControlNet"),ha.forEach(o),gn=i(bo,". To get better results, it is often helpful to:"),bo.forEach(o),qt=c(e),V=s(e,"OL",{});var yo=a(V);bt=s(yo,"LI",{});var _a=a(bt);hn=i(_a,"mask conditionings such that they don\u2019t overlap (for example, mask the area of a canny image where the pose conditioning is located)"),_a.forEach(o),_n=c(yo),pe=s(yo,"LI",{});var vo=a(pe);bn=i(vo,"experiment with the "),de=s(vo,"A",{href:!0,rel:!0});var ba=a(de);yt=s(ba,"CODE",{});var ya=a(yt);yn=i(ya,"controlnet_conditioning_scale"),ya.forEach(o),ba.forEach(o),vn=i(vo," parameter to determine how much weight to assign to each conditioning input"),vo.forEach(o),yo.forEach(o),Ot=c(e),Ve=s(e,"P",{});var va=a(Ve);wn=i(va,"In this example, you\u2019ll combine a canny image and a human pose estimation image to generate a new image."),va.forEach(o),Kt=c(e),Fe=s(e,"P",{});var wa=a(Fe);Mn=i(wa,"Prepare the canny image conditioning:"),wa.forEach(o),eo=c(e),_(ce.$$.fragment,e),to=c(e),X=s(e,"DIV",{class:!0});var wo=a(X);me=s(wo,"DIV",{});var Mo=a(me);Ye=s(Mo,"IMG",{class:!0,src:!0}),Tn=c(Mo),$e=s(Mo,"FIGCAPTION",{class:!0});var Ma=a($e);Jn=i(Ma,"original image"),Ma.forEach(o),Mo.forEach(o),Cn=c(wo),fe=s(wo,"DIV",{});var To=a(fe);Re=s(To,"IMG",{class:!0,src:!0}),Un=c(To),ze=s(To,"FIGCAPTION",{class:!0});var Ta=a(ze);xn=i(Ta,"canny image"),Ta.forEach(o),To.forEach(o),wo.forEach(o),oo=c(e),Qe=s(e,"P",{});var Ja=a(Qe);Nn=i(Ja,"Prepare the human pose estimation conditioning:"),Ja.forEach(o),no=c(e),_(ue.$$.fragment,e),so=c(e),L=s(e,"DIV",{class:!0});var Jo=a(L);ge=s(Jo,"DIV",{});var Co=a(ge);Ae=s(Co,"IMG",{class:!0,src:!0}),In=c(Co),He=s(Co,"FIGCAPTION",{class:!0});var Ca=a(He);jn=i(Ca,"original image"),Ca.forEach(o),Co.forEach(o),Zn=c(Jo),he=s(Jo,"DIV",{});var Uo=a(he);qe=s(Uo,"IMG",{class:!0,src:!0}),En=c(Uo),Oe=s(Uo,"FIGCAPTION",{class:!0});var Ua=a(Oe);Xn=i(Ua,"human pose image"),Ua.forEach(o),Uo.forEach(o),Jo.forEach(o),ao=c(e),I=s(e,"P",{});var lt=a(I);Ln=i(lt,"Load a list of ControlNet models that correspond to each conditioning, and pass them to the "),Ke=s(lt,"A",{href:!0});var xa=a(Ke);Sn=i(xa,"StableDiffusionXLControlNetPipeline"),xa.forEach(o),Wn=i(lt,". Use the faster "),et=s(lt,"A",{href:!0});var Na=a(et);kn=i(Na,"UniPCMultistepScheduler"),Na.forEach(o),Dn=i(lt," and nable model offloading to reduce memory usage."),lt.forEach(o),lo=c(e),_(_e.$$.fragment,e),io=c(e),tt=s(e,"P",{});var Ia=a(tt);Gn=i(Ia,"Now you can pass your prompt (an optional negative prompt if you\u2019re using one), canny image, and pose image to the pipeline:"),Ia.forEach(o),ro=c(e),_(be.$$.fragment,e),po=c(e),ye=s(e,"DIV",{class:!0});var ja=a(ye);ot=s(ja,"IMG",{class:!0,src:!0}),ja.forEach(o),co=c(e),S=s(e,"H2",{class:!0});var xo=a(S);F=s(xo,"A",{id:!0,class:!0,href:!0});var Za=a(F);vt=s(Za,"SPAN",{});var Ea=a(vt);_(ve.$$.fragment,Ea),Ea.forEach(o),Za.forEach(o),Pn=c(xo),wt=s(xo,"SPAN",{});var Xa=a(wt);Bn=i(Xa,"StableDiffusionXLControlNetPipeline"),Xa.forEach(o),xo.forEach(o),mo=c(e),f=s(e,"DIV",{class:!0});var u=a(f);_(we.$$.fragment,u),Vn=c(u),Mt=s(u,"P",{});var La=a(Mt);Fn=i(La,"Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance."),La.forEach(o),Yn=c(u),Me=s(u,"P",{});var No=a(Me);$n=i(No,"This model inherits from "),nt=s(No,"A",{href:!0});var Sa=a(nt);Rn=i(Sa,"DiffusionPipeline"),Sa.forEach(o),zn=i(No,`. Check the superclass documentation for the generic methods the
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)`),No.forEach(o),Qn=c(u),Tt=s(u,"P",{});var Wa=a(Tt);An=i(Wa,"In addition the pipeline inherits the following loading methods:"),Wa.forEach(o),Hn=c(u),Te=s(u,"UL",{});var Io=a(Te);Je=s(Io,"LI",{});var jo=a(Je);Jt=s(jo,"EM",{});var ka=a(Jt);qn=i(ka,"Textual-Inversion"),ka.forEach(o),On=i(jo,": "),st=s(jo,"A",{href:!0});var Da=a(st);Kn=i(Da,"loaders.TextualInversionLoaderMixin.load_textual_inversion()"),Da.forEach(o),jo.forEach(o),es=c(Io),Ce=s(Io,"LI",{});var Zo=a(Ce);Ct=s(Zo,"EM",{});var Ga=a(Ct);ts=i(Ga,"LoRA"),Ga.forEach(o),os=i(Zo,": "),at=s(Zo,"A",{href:!0});var Pa=a(at);ns=i(Pa,"loaders.LoraLoaderMixin.load_lora_weights()"),Pa.forEach(o),Zo.forEach(o),Io.forEach(o),ss=c(u),j=s(u,"DIV",{class:!0});var it=a(j);_(Ue.$$.fragment,it),as=c(it),Ut=s(it,"P",{});var Ba=a(Ut);ls=i(Ba,"Function invoked when calling the pipeline for generation."),Ba.forEach(o),is=c(it),_(Y.$$.fragment,it),it.forEach(o),rs=c(u),$=s(u,"DIV",{class:!0});var Eo=a($);_(xe.$$.fragment,Eo),ps=c(Eo),Ne=s(Eo,"P",{});var Xo=a(Ne);ds=i(Xo,"Disable sliced VAE decoding. If "),xt=s(Xo,"CODE",{});var Va=a(xt);cs=i(Va,"enable_vae_slicing"),Va.forEach(o),ms=i(Xo,` was previously enabled, this method will go back to
computing decoding in one step.`),Xo.forEach(o),Eo.forEach(o),fs=c(u),R=s(u,"DIV",{class:!0});var Lo=a(R);_(Ie.$$.fragment,Lo),us=c(Lo),je=s(Lo,"P",{});var So=a(je);gs=i(So,"Disable tiled VAE decoding. If "),Nt=s(So,"CODE",{});var Fa=a(Nt);hs=i(Fa,"enable_vae_tiling"),Fa.forEach(o),_s=i(So,` was previously enabled, this method will go back to
computing decoding in one step.`),So.forEach(o),Lo.forEach(o),bs=c(u),z=s(u,"DIV",{class:!0});var Wo=a(z);_(Ze.$$.fragment,Wo),ys=c(Wo),C=s(Wo,"P",{});var Z=a(C);vs=i(Z,`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),It=s(Z,"CODE",{});var Ya=a(It);ws=i(Ya,"enable_sequential_cpu_offload"),Ya.forEach(o),Ms=i(Z,", this method moves one whole model at a time to the GPU when its "),jt=s(Z,"CODE",{});var $a=a(jt);Ts=i($a,"forward"),$a.forEach(o),Js=i(Z,`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Zt=s(Z,"CODE",{});var Ra=a(Zt);Cs=i(Ra,"enable_sequential_cpu_offload"),Ra.forEach(o),Us=i(Z,", but performance is much better due to the iterative execution of the "),Et=s(Z,"CODE",{});var za=a(Et);xs=i(za,"unet"),za.forEach(o),Ns=i(Z,"."),Z.forEach(o),Wo.forEach(o),Is=c(u),Q=s(u,"DIV",{class:!0});var ko=a(Q);_(Ee.$$.fragment,ko),js=c(ko),Xt=s(ko,"P",{});var Qa=a(Xt);Zs=i(Qa,`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),Qa.forEach(o),ko.forEach(o),Es=c(u),A=s(u,"DIV",{class:!0});var Do=a(A);_(Xe.$$.fragment,Do),Xs=c(Do),Lt=s(Do,"P",{});var Aa=a(Lt);Ls=i(Aa,`Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
processing larger images.`),Aa.forEach(o),Do.forEach(o),Ss=c(u),H=s(u,"DIV",{class:!0});var Go=a(H);_(Le.$$.fragment,Go),Ws=c(Go),St=s(Go,"P",{});var Ha=a(St);ks=i(Ha,"Encodes the prompt into text encoder hidden states."),Ha.forEach(o),Go.forEach(o),u.forEach(o),this.h()},h(){p(M,"name","hf:doc:metadata"),p(M,"content",JSON.stringify(ll)),p(T,"id","controlnet-with-stable-diffusion-xl"),p(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(T,"href","#controlnet-with-stable-diffusion-xl"),p(J,"class","relative group"),p(ee,"href","https://huggingface.co/papers/2302.05543"),p(ee,"rel","nofollow"),p(Ge,"href","./stable_diffusion/stable_diffusion_xl.md"),p(te,"href","https://huggingface.co/models?other=stable-diffusion-xl&other=controlnet"),p(te,"rel","nofollow"),p(oe,"href","https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0-small"),p(oe,"rel","nofollow"),p(ne,"href","https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0-mid"),p(ne,"rel","nofollow"),p(se,"href","https://huggingface.co/diffusers/controlnet-depth-sdxl-1.0-small"),p(se,"rel","nofollow"),p(ae,"href","https://huggingface.co/diffusers/controlnet-depth-sdxl-1.0-mid"),p(ae,"rel","nofollow"),p(le,"href","https://github.com/huggingface/diffusers/blob/main/examples/controlnet/README_sdxl.md"),p(le,"rel","nofollow"),rt(ie.src,Ps="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/sd_xl/sdxl_controlnet_canny_grid.png")||p(ie,"src",Ps),p(ie,"width","600"),p(P,"id","multicontrolnet"),p(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(P,"href","#multicontrolnet"),p(E,"class","relative group"),p(de,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/controlnet#diffusers.StableDiffusionControlNetPipeline.__call__.controlnet_conditioning_scale"),p(de,"rel","nofollow"),p(Ye,"class","rounded-xl"),rt(Ye.src,Bs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/landscape.png")||p(Ye,"src",Bs),p($e,"class","mt-2 text-center text-sm text-gray-500"),p(Re,"class","rounded-xl"),rt(Re.src,Vs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/controlnet/landscape_canny_masked.png")||p(Re,"src",Vs),p(ze,"class","mt-2 text-center text-sm text-gray-500"),p(X,"class","flex gap-4"),p(Ae,"class","rounded-xl"),rt(Ae.src,Fs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/person.png")||p(Ae,"src",Fs),p(He,"class","mt-2 text-center text-sm text-gray-500"),p(qe,"class","rounded-xl"),rt(qe.src,Ys="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/controlnet/person_pose.png")||p(qe,"src",Ys),p(Oe,"class","mt-2 text-center text-sm text-gray-500"),p(L,"class","flex gap-4"),p(Ke,"href","/docs/diffusers/v0.20.0/en/api/pipelines/controlnet_sdxl#diffusers.StableDiffusionXLControlNetPipeline"),p(et,"href","/docs/diffusers/v0.20.0/en/api/schedulers/unipc#diffusers.UniPCMultistepScheduler"),p(ot,"class","rounded-xl"),rt(ot.src,$s="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/multicontrolnet.png")||p(ot,"src",$s),p(ye,"class","flex justify-center"),p(F,"id","diffusers.StableDiffusionXLControlNetPipeline"),p(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(F,"href","#diffusers.StableDiffusionXLControlNetPipeline"),p(S,"class","relative group"),p(nt,"href","/docs/diffusers/v0.20.0/en/api/pipelines/overview#diffusers.DiffusionPipeline"),p(st,"href","/docs/diffusers/v0.20.0/en/api/pipelines/stable_diffusion/depth2img#diffusers.StableDiffusionDepth2ImgPipeline.load_textual_inversion"),p(at,"href","/docs/diffusers/v0.20.0/en/api/pipelines/stable_diffusion/depth2img#diffusers.StableDiffusionDepth2ImgPipeline.load_lora_weights"),p(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){t(document.head,M),m(e,W,r),m(e,J,r),t(J,T),t(T,x),b(g,x,null),t(J,N),t(J,pt),t(pt,Po),m(e,Dt,r),m(e,K,r),t(K,ee),t(ee,Bo),t(K,Vo),m(e,Gt,r),m(e,We,r),t(We,Fo),m(e,Pt,r),m(e,ke,r),t(ke,Yo),m(e,Bt,r),m(e,De,r),t(De,dt),t(dt,$o),m(e,Vt,r),m(e,k,r),t(k,Ro),t(k,Ge),t(Ge,zo),t(k,Qo),m(e,Ft,r),m(e,D,r),t(D,Ao),t(D,te),t(te,Ho),t(D,qo),m(e,Yt,r),m(e,U,r),t(U,ct),t(ct,oe),t(oe,Oo),t(U,Ko),t(U,mt),t(mt,ne),t(ne,en),t(U,tn),t(U,ft),t(ft,se),t(se,on),t(U,nn),t(U,ut),t(ut,ae),t(ae,sn),m(e,$t,r),m(e,G,r),t(G,an),t(G,le),t(le,ln),t(G,rn),m(e,Rt,r),m(e,Pe,r),t(Pe,pn),m(e,zt,r),m(e,ie,r),m(e,Qt,r),m(e,Be,r),t(Be,dn),m(e,At,r),m(e,E,r),t(E,P),t(P,gt),b(re,gt,null),t(E,cn),t(E,ht),t(ht,mn),m(e,Ht,r),m(e,B,r),t(B,fn),t(B,_t),t(_t,un),t(B,gn),m(e,qt,r),m(e,V,r),t(V,bt),t(bt,hn),t(V,_n),t(V,pe),t(pe,bn),t(pe,de),t(de,yt),t(yt,yn),t(pe,vn),m(e,Ot,r),m(e,Ve,r),t(Ve,wn),m(e,Kt,r),m(e,Fe,r),t(Fe,Mn),m(e,eo,r),b(ce,e,r),m(e,to,r),m(e,X,r),t(X,me),t(me,Ye),t(me,Tn),t(me,$e),t($e,Jn),t(X,Cn),t(X,fe),t(fe,Re),t(fe,Un),t(fe,ze),t(ze,xn),m(e,oo,r),m(e,Qe,r),t(Qe,Nn),m(e,no,r),b(ue,e,r),m(e,so,r),m(e,L,r),t(L,ge),t(ge,Ae),t(ge,In),t(ge,He),t(He,jn),t(L,Zn),t(L,he),t(he,qe),t(he,En),t(he,Oe),t(Oe,Xn),m(e,ao,r),m(e,I,r),t(I,Ln),t(I,Ke),t(Ke,Sn),t(I,Wn),t(I,et),t(et,kn),t(I,Dn),m(e,lo,r),b(_e,e,r),m(e,io,r),m(e,tt,r),t(tt,Gn),m(e,ro,r),b(be,e,r),m(e,po,r),m(e,ye,r),t(ye,ot),m(e,co,r),m(e,S,r),t(S,F),t(F,vt),b(ve,vt,null),t(S,Pn),t(S,wt),t(wt,Bn),m(e,mo,r),m(e,f,r),b(we,f,null),t(f,Vn),t(f,Mt),t(Mt,Fn),t(f,Yn),t(f,Me),t(Me,$n),t(Me,nt),t(nt,Rn),t(Me,zn),t(f,Qn),t(f,Tt),t(Tt,An),t(f,Hn),t(f,Te),t(Te,Je),t(Je,Jt),t(Jt,qn),t(Je,On),t(Je,st),t(st,Kn),t(Te,es),t(Te,Ce),t(Ce,Ct),t(Ct,ts),t(Ce,os),t(Ce,at),t(at,ns),t(f,ss),t(f,j),b(Ue,j,null),t(j,as),t(j,Ut),t(Ut,ls),t(j,is),b(Y,j,null),t(f,rs),t(f,$),b(xe,$,null),t($,ps),t($,Ne),t(Ne,ds),t(Ne,xt),t(xt,cs),t(Ne,ms),t(f,fs),t(f,R),b(Ie,R,null),t(R,us),t(R,je),t(je,gs),t(je,Nt),t(Nt,hs),t(je,_s),t(f,bs),t(f,z),b(Ze,z,null),t(z,ys),t(z,C),t(C,vs),t(C,It),t(It,ws),t(C,Ms),t(C,jt),t(jt,Ts),t(C,Js),t(C,Zt),t(Zt,Cs),t(C,Us),t(C,Et),t(Et,xs),t(C,Ns),t(f,Is),t(f,Q),b(Ee,Q,null),t(Q,js),t(Q,Xt),t(Xt,Zs),t(f,Es),t(f,A),b(Xe,A,null),t(A,Xs),t(A,Lt),t(Lt,Ls),t(f,Ss),t(f,H),b(Le,H,null),t(H,Ws),t(H,St),t(St,ks),fo=!0},p(e,[r]){const Se={};r&2&&(Se.$$scope={dirty:r,ctx:e}),Y.$set(Se)},i(e){fo||(y(g.$$.fragment,e),y(re.$$.fragment,e),y(ce.$$.fragment,e),y(ue.$$.fragment,e),y(_e.$$.fragment,e),y(be.$$.fragment,e),y(ve.$$.fragment,e),y(we.$$.fragment,e),y(Ue.$$.fragment,e),y(Y.$$.fragment,e),y(xe.$$.fragment,e),y(Ie.$$.fragment,e),y(Ze.$$.fragment,e),y(Ee.$$.fragment,e),y(Xe.$$.fragment,e),y(Le.$$.fragment,e),fo=!0)},o(e){v(g.$$.fragment,e),v(re.$$.fragment,e),v(ce.$$.fragment,e),v(ue.$$.fragment,e),v(_e.$$.fragment,e),v(be.$$.fragment,e),v(ve.$$.fragment,e),v(we.$$.fragment,e),v(Ue.$$.fragment,e),v(Y.$$.fragment,e),v(xe.$$.fragment,e),v(Ie.$$.fragment,e),v(Ze.$$.fragment,e),v(Ee.$$.fragment,e),v(Xe.$$.fragment,e),v(Le.$$.fragment,e),fo=!1},d(e){o(M),e&&o(W),e&&o(J),w(g),e&&o(Dt),e&&o(K),e&&o(Gt),e&&o(We),e&&o(Pt),e&&o(ke),e&&o(Bt),e&&o(De),e&&o(Vt),e&&o(k),e&&o(Ft),e&&o(D),e&&o(Yt),e&&o(U),e&&o($t),e&&o(G),e&&o(Rt),e&&o(Pe),e&&o(zt),e&&o(ie),e&&o(Qt),e&&o(Be),e&&o(At),e&&o(E),w(re),e&&o(Ht),e&&o(B),e&&o(qt),e&&o(V),e&&o(Ot),e&&o(Ve),e&&o(Kt),e&&o(Fe),e&&o(eo),w(ce,e),e&&o(to),e&&o(X),e&&o(oo),e&&o(Qe),e&&o(no),w(ue,e),e&&o(so),e&&o(L),e&&o(ao),e&&o(I),e&&o(lo),w(_e,e),e&&o(io),e&&o(tt),e&&o(ro),w(be,e),e&&o(po),e&&o(ye),e&&o(co),e&&o(S),w(ve),e&&o(mo),e&&o(f),w(we),w(Ue),w(Y),w(xe),w(Ie),w(Ze),w(Ee),w(Xe),w(Le)}}}const ll={local:"controlnet-with-stable-diffusion-xl",sections:[{local:"multicontrolnet",title:"MultiControlNet"},{local:"diffusers.StableDiffusionXLControlNetPipeline",title:"StableDiffusionXLControlNetPipeline"}],title:"ControlNet with Stable Diffusion XL"};function il(kt){return tl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fl extends qa{constructor(M){super();Oa(this,M,il,al,Ka,{})}}export{fl as default,ll as metadata};
