import{s as st,a as lt,n as at,o as ot}from"../chunks/scheduler.ef843396.js";import{S as rt,i as ut,e as a,s,c as X,h as mt,a as o,d as i,b as l,f as Y,g as Z,j as u,k as G,l as ct,m as n,n as tt,t as et,o as it,p as nt}from"../chunks/index.05ef1181.js";import{H as pt,E as ht}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ef6649bf.js";function dt(U){let r,L,M,H,m,P,c,W,x,p,j="In unit 4, we learned about our first Policy-Based algorithm called <strong>Reinforce</strong>.",C,h,O="In Policy-Based methods, <strong>we aim to optimize the policy directly without using a value function</strong>. More precisely, Reinforce is part of a subclass of <em>Policy-Based Methods</em> called <em>Policy-Gradient methods</em>. This subclass optimizes the policy directly by <strong>estimating the weights of the optimal policy using Gradient Ascent</strong>.",$,d,V="We saw that Reinforce worked well. However, because we use Monte-Carlo sampling to estimate return (we use an entire episode to calculate the return), <strong>we have significant variance in policy gradient estimation</strong>.",A,g,D="Remember that the policy gradient estimation is <strong>the direction of the steepest increase in return</strong>. In other words, how to update our policy weights so that actions that lead to good returns have a higher probability of being taken. The Monte Carlo variance, which we will further study in this unit, <strong>leads to slower training since we need a lot of samples to mitigate it</strong>.",B,f,F="So today we’ll study <strong>Actor-Critic methods</strong>, a hybrid architecture combining value-based and Policy-Based methods that helps to stabilize the training by reducing the variance using:",k,v,J="<li><em>An Actor</em> that controls <strong>how our agent behaves</strong> (Policy-Based method)</li> <li><em>A Critic</em> that measures <strong>how good the taken action is</strong> (Value-Based method)</li>",z,b,K="We’ll study one of these hybrid methods, Advantage Actor Critic (A2C), <strong>and train our agent using Stable-Baselines3 in robotic environments</strong>. We’ll train:",E,_,N="<li>A robotic arm 🦾 to move to the correct position.</li>",I,w,Q="Sound exciting? Let’s get started!",S,y,R,T,q;return m=new pt({props:{title:"Introduction",local:"introduction",headingTag:"h1"}}),y=new ht({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit6/introduction.mdx"}}),{c(){r=a("meta"),L=s(),M=a("p"),H=s(),X(m.$$.fragment),P=s(),c=a("img"),x=s(),p=a("p"),p.innerHTML=j,C=s(),h=a("p"),h.innerHTML=O,$=s(),d=a("p"),d.innerHTML=V,A=s(),g=a("p"),g.innerHTML=D,B=s(),f=a("p"),f.innerHTML=F,k=s(),v=a("ul"),v.innerHTML=J,z=s(),b=a("p"),b.innerHTML=K,E=s(),_=a("ul"),_.innerHTML=N,I=s(),w=a("p"),w.textContent=Q,S=s(),X(y.$$.fragment),R=s(),T=a("p"),this.h()},l(t){const e=mt("svelte-u9bgzb",document.head);r=o(e,"META",{name:!0,content:!0}),e.forEach(i),L=l(t),M=o(t,"P",{}),Y(M).forEach(i),H=l(t),Z(m.$$.fragment,t),P=l(t),c=o(t,"IMG",{src:!0,alt:!0}),x=l(t),p=o(t,"P",{"data-svelte-h":!0}),u(p)!=="svelte-dvu1gd"&&(p.innerHTML=j),C=l(t),h=o(t,"P",{"data-svelte-h":!0}),u(h)!=="svelte-z9stkq"&&(h.innerHTML=O),$=l(t),d=o(t,"P",{"data-svelte-h":!0}),u(d)!=="svelte-1p5t49p"&&(d.innerHTML=V),A=l(t),g=o(t,"P",{"data-svelte-h":!0}),u(g)!=="svelte-spae8s"&&(g.innerHTML=D),B=l(t),f=o(t,"P",{"data-svelte-h":!0}),u(f)!=="svelte-a75o96"&&(f.innerHTML=F),k=l(t),v=o(t,"UL",{"data-svelte-h":!0}),u(v)!=="svelte-1g2297h"&&(v.innerHTML=J),z=l(t),b=o(t,"P",{"data-svelte-h":!0}),u(b)!=="svelte-54x6jn"&&(b.innerHTML=K),E=l(t),_=o(t,"UL",{"data-svelte-h":!0}),u(_)!=="svelte-10o7xql"&&(_.innerHTML=N),I=l(t),w=o(t,"P",{"data-svelte-h":!0}),u(w)!=="svelte-1lrk2z4"&&(w.textContent=Q),S=l(t),Z(y.$$.fragment,t),R=l(t),T=o(t,"P",{}),Y(T).forEach(i),this.h()},h(){G(r,"name","hf:doc:metadata"),G(r,"content",gt),lt(c.src,W="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/thumbnail.png")||G(c,"src",W),G(c,"alt","Thumbnail")},m(t,e){ct(document.head,r),n(t,L,e),n(t,M,e),n(t,H,e),tt(m,t,e),n(t,P,e),n(t,c,e),n(t,x,e),n(t,p,e),n(t,C,e),n(t,h,e),n(t,$,e),n(t,d,e),n(t,A,e),n(t,g,e),n(t,B,e),n(t,f,e),n(t,k,e),n(t,v,e),n(t,z,e),n(t,b,e),n(t,E,e),n(t,_,e),n(t,I,e),n(t,w,e),n(t,S,e),tt(y,t,e),n(t,R,e),n(t,T,e),q=!0},p:at,i(t){q||(et(m.$$.fragment,t),et(y.$$.fragment,t),q=!0)},o(t){it(m.$$.fragment,t),it(y.$$.fragment,t),q=!1},d(t){t&&(i(L),i(M),i(H),i(P),i(c),i(x),i(p),i(C),i(h),i($),i(d),i(A),i(g),i(B),i(f),i(k),i(v),i(z),i(b),i(E),i(_),i(I),i(w),i(S),i(R),i(T)),i(r),nt(m,t),nt(y,t)}}}const gt='{"title":"Introduction","local":"introduction","sections":[],"depth":1}';function ft(U){return ot(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class wt extends rt{constructor(r){super(),ut(this,r,ft,dt,st,{})}}export{wt as component};
