import{s as q,n as F,o as G}from"../chunks/scheduler.37c15a92.js";import{S as N,i as O,g as c,s as i,r as T,A as R,h as $,f as a,c as l,j as B,u as y,x as z,k as W,y as D,a as n,v as k,d as H,t as L,w as M}from"../chunks/index.2bf4358c.js";import{C as I}from"../chunks/CourseFloatingBanner.9ff4c771.js";import{H as J,E as K}from"../chunks/getInferenceSnippets.1837c472.js";function Q(S){let s,d,h,g,r,v,m,x,o,U="完成這一章，辛苦了！",C,p,j="在深入研究標記器之後，您應該：",P,f,A="<li>能夠使用舊的標記器作為模板來訓練新的標記器</li> <li>瞭解如何使用偏移量將標記的位置映射到其原始文本範圍</li> <li>瞭解 BPE、WordPiece 和 Unigram 之間的區別</li> <li>能夠混合和匹配 🤗 Tokenizers 庫提供的塊來構建您自己的標記器</li> <li>能夠在 🤗 Transformers 庫中使用該標記器</li>",b,u,w,_,E;return r=new J({props:{title:"標記器，回顧！",local:"標記器回顧",headingTag:"h1"}}),m=new I({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),u=new K({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-TW/chapter6/9.mdx"}}),{c(){s=c("meta"),d=i(),h=c("p"),g=i(),T(r.$$.fragment),v=i(),T(m.$$.fragment),x=i(),o=c("p"),o.textContent=U,C=i(),p=c("p"),p.textContent=j,P=i(),f=c("ul"),f.innerHTML=A,b=i(),T(u.$$.fragment),w=i(),_=c("p"),this.h()},l(t){const e=R("svelte-u9bgzb",document.head);s=$(e,"META",{name:!0,content:!0}),e.forEach(a),d=l(t),h=$(t,"P",{}),B(h).forEach(a),g=l(t),y(r.$$.fragment,t),v=l(t),y(m.$$.fragment,t),x=l(t),o=$(t,"P",{"data-svelte-h":!0}),z(o)!=="svelte-1lh8k2v"&&(o.textContent=U),C=l(t),p=$(t,"P",{"data-svelte-h":!0}),z(p)!=="svelte-kvjzhn"&&(p.textContent=j),P=l(t),f=$(t,"UL",{"data-svelte-h":!0}),z(f)!=="svelte-5ywe36"&&(f.innerHTML=A),b=l(t),y(u.$$.fragment,t),w=l(t),_=$(t,"P",{}),B(_).forEach(a),this.h()},h(){W(s,"name","hf:doc:metadata"),W(s,"content",V)},m(t,e){D(document.head,s),n(t,d,e),n(t,h,e),n(t,g,e),k(r,t,e),n(t,v,e),k(m,t,e),n(t,x,e),n(t,o,e),n(t,C,e),n(t,p,e),n(t,P,e),n(t,f,e),n(t,b,e),k(u,t,e),n(t,w,e),n(t,_,e),E=!0},p:F,i(t){E||(H(r.$$.fragment,t),H(m.$$.fragment,t),H(u.$$.fragment,t),E=!0)},o(t){L(r.$$.fragment,t),L(m.$$.fragment,t),L(u.$$.fragment,t),E=!1},d(t){t&&(a(d),a(h),a(g),a(v),a(x),a(o),a(C),a(p),a(P),a(f),a(b),a(w),a(_)),a(s),M(r,t),M(m,t),M(u,t)}}}const V='{"title":"標記器，回顧！","local":"標記器回顧","sections":[],"depth":1}';function X(S){return G(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class at extends N{constructor(s){super(),O(this,s,X,Q,q,{})}}export{at as component};
