import{s as et,n as at,o as st}from"../chunks/scheduler.d6170356.js";import{S as lt,i as nt,g as d,s as l,r as k,A as it,h as r,f as a,c as n,j as O,u as Z,x as U,k as tt,y as dt,a as s,v as _,d as R,t as T,w as X}from"../chunks/index.fcd4cc08.js";import{C as N}from"../chunks/CodeBlock.7b16bdef.js";import{H as rt,E as ot}from"../chunks/EditOnGithub.da2b595c.js";function pt(L){let i,w,b,j,o,Y,p,S=`<a href="https://github.com/dask/dask" rel="nofollow">Dask</a> is a parallel and distributed computing library that scales the existing Python and PyData ecosystem.
Since it uses <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="/docs/huggingface_hub/guides/hf_file_system#integrations"><code>hf://</code></a>) to read and write data on the Hub:`,B,h,x='First you need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',F,m,q,c,P='Then you can <a href="/docs/huggingface_hub/quick-start#create-a-repository">Create a dataset repository</a>, for example using:',v,f,H,u,D='Finally, you can use <a href="/docs/huggingface_hub/guides/hf_file_system#integrations">Hugging Face paths</a> in Dask:',V,y,z,M,A=`This creates a dataset repository <code>username/my_dataset</code> containing your Dask dataset in Parquet format.
You can reload it later:`,C,J,W,g,K='For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="/docs/huggingface_hub/guides/hf_file_system">the client library’s documentation on the HfFileSystem</a>.',I,G,Q,$,E;return o=new rt({props:{title:"Dask",local:"dask",headingTag:"h1"}}),m=new N({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),f=new N({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBJTBBSGZBcGkoKS5jcmVhdGVfcmVwbyhyZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyJTJDJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMik=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi

HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)`,wrap:!1}}),y=new N({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyKSUwQSUwQSUyMyUyMG9yJTIwd3JpdGUlMjBpbiUyMHNlcGFyYXRlJTIwZGlyZWN0b3JpZXMlMjBpZiUyMHRoZSUyMGRhdGFzZXQlMjBoYXMlMjB0cmFpbiUyRnZhbGlkYXRpb24lMkZ0ZXN0JTIwc3BsaXRzJTBBZGZfdHJhaW4udG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdmFsaWRhdGlvbiUyMiklMEFkZl90ZXN0JTIwLnRvX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnRlc3QlMjIp",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or write in separate directories if the dataset has train/validation/test splits</span>
df_train.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test .to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),J=new N({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyMiklMEElMEElMjMlMjBvciUyMHJlYWQlMjBmcm9tJTIwc2VwYXJhdGUlMjBkaXJlY3RvcmllcyUyMGlmJTIwdGhlJTIwZGF0YXNldCUyMGhhcyUyMHRyYWluJTJGdmFsaWRhdGlvbiUyRnRlc3QlMjBzcGxpdHMlMEFkZl90cmFpbiUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnZhbGlkYXRpb24lMjIpJTBBZGZfdGVzdCUyMCUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdGVzdCUyMik=",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or read from separate directories if the dataset has train/validation/test splits</span>
df_train = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test  = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),G=new ot({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-dask.md"}}),{c(){i=d("meta"),w=l(),b=d("p"),j=l(),k(o.$$.fragment),Y=l(),p=d("p"),p.innerHTML=S,B=l(),h=d("p"),h.innerHTML=x,F=l(),k(m.$$.fragment),q=l(),c=d("p"),c.innerHTML=P,v=l(),k(f.$$.fragment),H=l(),u=d("p"),u.innerHTML=D,V=l(),k(y.$$.fragment),z=l(),M=d("p"),M.innerHTML=A,C=l(),k(J.$$.fragment),W=l(),g=d("p"),g.innerHTML=K,I=l(),k(G.$$.fragment),Q=l(),$=d("p"),this.h()},l(t){const e=it("svelte-u9bgzb",document.head);i=r(e,"META",{name:!0,content:!0}),e.forEach(a),w=n(t),b=r(t,"P",{}),O(b).forEach(a),j=n(t),Z(o.$$.fragment,t),Y=n(t),p=r(t,"P",{"data-svelte-h":!0}),U(p)!=="svelte-1i0h8ri"&&(p.innerHTML=S),B=n(t),h=r(t,"P",{"data-svelte-h":!0}),U(h)!=="svelte-1jdzygp"&&(h.innerHTML=x),F=n(t),Z(m.$$.fragment,t),q=n(t),c=r(t,"P",{"data-svelte-h":!0}),U(c)!=="svelte-wfb25d"&&(c.innerHTML=P),v=n(t),Z(f.$$.fragment,t),H=n(t),u=r(t,"P",{"data-svelte-h":!0}),U(u)!=="svelte-1vftqwj"&&(u.innerHTML=D),V=n(t),Z(y.$$.fragment,t),z=n(t),M=r(t,"P",{"data-svelte-h":!0}),U(M)!=="svelte-14rczfs"&&(M.innerHTML=A),C=n(t),Z(J.$$.fragment,t),W=n(t),g=r(t,"P",{"data-svelte-h":!0}),U(g)!=="svelte-smuvdf"&&(g.innerHTML=K),I=n(t),Z(G.$$.fragment,t),Q=n(t),$=r(t,"P",{}),O($).forEach(a),this.h()},h(){tt(i,"name","hf:doc:metadata"),tt(i,"content",ht)},m(t,e){dt(document.head,i),s(t,w,e),s(t,b,e),s(t,j,e),_(o,t,e),s(t,Y,e),s(t,p,e),s(t,B,e),s(t,h,e),s(t,F,e),_(m,t,e),s(t,q,e),s(t,c,e),s(t,v,e),_(f,t,e),s(t,H,e),s(t,u,e),s(t,V,e),_(y,t,e),s(t,z,e),s(t,M,e),s(t,C,e),_(J,t,e),s(t,W,e),s(t,g,e),s(t,I,e),_(G,t,e),s(t,Q,e),s(t,$,e),E=!0},p:at,i(t){E||(R(o.$$.fragment,t),R(m.$$.fragment,t),R(f.$$.fragment,t),R(y.$$.fragment,t),R(J.$$.fragment,t),R(G.$$.fragment,t),E=!0)},o(t){T(o.$$.fragment,t),T(m.$$.fragment,t),T(f.$$.fragment,t),T(y.$$.fragment,t),T(J.$$.fragment,t),T(G.$$.fragment,t),E=!1},d(t){t&&(a(w),a(b),a(j),a(Y),a(p),a(B),a(h),a(F),a(q),a(c),a(v),a(H),a(u),a(V),a(z),a(M),a(C),a(W),a(g),a(I),a(Q),a($)),a(i),X(o,t),X(m,t),X(f,t),X(y,t),X(J,t),X(G,t)}}}const ht='{"title":"Dask","local":"dask","sections":[],"depth":1}';function mt(L){return st(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mt extends lt{constructor(i){super(),nt(this,i,mt,pt,et,{})}}export{Mt as component};
