import{s as le,a as V,n as se,o as oe}from"../chunks/scheduler.ef843396.js";import{S as re,i as ue,e as s,s as a,c as X,h as me,a as o,d as n,b as l,f as Z,g as ee,j as m,k as y,l as ce,m as i,n as te,t as ne,o as ie,p as ae}from"../chunks/index.05ef1181.js";import{H as pe,E as fe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ef6649bf.js";function ge(U){let r,T,$,C,c,L,p,A,P,f,D="In the last unit, we learned about Deep Q-Learning. In this value-based deep reinforcement learning algorithm, we <strong>used a deep neural network to approximate the different Q-values for each possible action at a state.</strong>",H,g,F="Since the beginning of the course, we have only studied value-based methods, <strong>where we estimate a value function as an intermediate step towards finding an optimal policy.</strong>",E,d,O,I,h,W="In value-based methods, the policy <strong>(π) only exists because of the action value estimates since the policy is just a function</strong> (for instance, greedy-policy) that will select the action with the highest value given a state.",k,v,Y="With policy-based methods, we want to optimize the policy directly <strong>without having an intermediate step of learning a value function.</strong>",j,_,B=`So today, <strong>we’ll learn about policy-based methods and study a subset of these methods called policy gradient</strong>. Then we’ll implement our first policy gradient algorithm called Monte Carlo <strong>Reinforce</strong> from scratch using PyTorch.
Then, we’ll test its robustness using the CartPole-v1 and PixelCopter environments.`,S,b,J="You’ll then be able to iterate and improve this implementation for more advanced environments.",q,u,K='<img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit6/envs.gif" alt="Environments"/>',G,x,N="Let’s get started!",z,w,R,M,Q;return c=new pe({props:{title:"Introduction",local:"introduction",headingTag:"h1"}}),w=new fe({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit4/introduction.mdx"}}),{c(){r=s("meta"),T=a(),$=s("p"),C=a(),X(c.$$.fragment),L=a(),p=s("img"),P=a(),f=s("p"),f.innerHTML=D,H=a(),g=s("p"),g.innerHTML=F,E=a(),d=s("img"),I=a(),h=s("p"),h.innerHTML=W,k=a(),v=s("p"),v.innerHTML=Y,j=a(),_=s("p"),_.innerHTML=B,S=a(),b=s("p"),b.textContent=J,q=a(),u=s("figure"),u.innerHTML=K,G=a(),x=s("p"),x.textContent=N,z=a(),X(w.$$.fragment),R=a(),M=s("p"),this.h()},l(e){const t=me("svelte-u9bgzb",document.head);r=o(t,"META",{name:!0,content:!0}),t.forEach(n),T=l(e),$=o(e,"P",{}),Z($).forEach(n),C=l(e),ee(c.$$.fragment,e),L=l(e),p=o(e,"IMG",{src:!0,alt:!0}),P=l(e),f=o(e,"P",{"data-svelte-h":!0}),m(f)!=="svelte-j9zk2f"&&(f.innerHTML=D),H=l(e),g=o(e,"P",{"data-svelte-h":!0}),m(g)!=="svelte-2xpx7a"&&(g.innerHTML=F),E=l(e),d=o(e,"IMG",{src:!0,alt:!0}),I=l(e),h=o(e,"P",{"data-svelte-h":!0}),m(h)!=="svelte-10c158u"&&(h.innerHTML=W),k=l(e),v=o(e,"P",{"data-svelte-h":!0}),m(v)!=="svelte-1btggbm"&&(v.innerHTML=Y),j=l(e),_=o(e,"P",{"data-svelte-h":!0}),m(_)!=="svelte-4of5it"&&(_.innerHTML=B),S=l(e),b=o(e,"P",{"data-svelte-h":!0}),m(b)!=="svelte-liqaak"&&(b.textContent=J),q=l(e),u=o(e,"FIGURE",{class:!0,"data-svelte-h":!0}),m(u)!=="svelte-1dadvfq"&&(u.innerHTML=K),G=l(e),x=o(e,"P",{"data-svelte-h":!0}),m(x)!=="svelte-4b3xjd"&&(x.textContent=N),z=l(e),ee(w.$$.fragment,e),R=l(e),M=o(e,"P",{}),Z(M).forEach(n),this.h()},h(){y(r,"name","hf:doc:metadata"),y(r,"content",de),V(p.src,A="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit6/thumbnail.png")||y(p,"src",A),y(p,"alt","thumbnail"),V(d.src,O="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit3/link-value-policy.jpg")||y(d,"src",O),y(d,"alt","Link value policy"),y(u,"class","image table text-center m-0 w-full")},m(e,t){ce(document.head,r),i(e,T,t),i(e,$,t),i(e,C,t),te(c,e,t),i(e,L,t),i(e,p,t),i(e,P,t),i(e,f,t),i(e,H,t),i(e,g,t),i(e,E,t),i(e,d,t),i(e,I,t),i(e,h,t),i(e,k,t),i(e,v,t),i(e,j,t),i(e,_,t),i(e,S,t),i(e,b,t),i(e,q,t),i(e,u,t),i(e,G,t),i(e,x,t),i(e,z,t),te(w,e,t),i(e,R,t),i(e,M,t),Q=!0},p:se,i(e){Q||(ne(c.$$.fragment,e),ne(w.$$.fragment,e),Q=!0)},o(e){ie(c.$$.fragment,e),ie(w.$$.fragment,e),Q=!1},d(e){e&&(n(T),n($),n(C),n(L),n(p),n(P),n(f),n(H),n(g),n(E),n(d),n(I),n(h),n(k),n(v),n(j),n(_),n(S),n(b),n(q),n(u),n(G),n(x),n(z),n(R),n(M)),n(r),ae(c,e),ae(w,e)}}}const de='{"title":"Introduction","local":"introduction","sections":[],"depth":1}';function he(U){return oe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xe extends re{constructor(r){super(),ue(this,r,he,ge,le,{})}}export{xe as component};
