import{s as Ts,n as rs,o as ws}from"../chunks/scheduler.d6170356.js";import{S as hs,i as js,g as M,s as t,r as J,A as os,h as p,f as a,c as e,j as cs,u as U,x as y,k as vl,y as ms,a as n,v as i,d as c,t as T,w as r}from"../chunks/index.fcd4cc08.js";import{C as w}from"../chunks/CodeBlock.7b16bdef.js";import{H as al,E as us}from"../chunks/EditOnGithub.da2b595c.js";function ds(_l){let h,nl,ll,tl,m,el,u,ql="Spark enables real-time, large-scale data processing in a distributed environment.",Ml,d,Hl="In particular you can use <code>huggingface_hub</code> to access Hugging Face datasets repositories in PySpark",pl,I,yl,b,Dl="To be able to read and write to Hugging Face URLs (e.g. <code>hf://datasets/username/dataset/data.parquet</code>), you need to install the <code>huggingface_hub</code> library:",Jl,C,Ul,B,xl="You also need to install <code>pyarrow</code> to read/write Parquet / JSON / CSV / etc. files using the filesystem API provided by <code>huggingFace_hub</code>:",il,g,cl,G,Tl,f,Ol="You need to authenticate to Hugging Face to read private/gated dataset repositories or to write to your dataset repositories.",rl,k,Ll="You can use the CLI for example:",wl,Z,hl,R,$l="It’s also possible to provide your Hugging Face token with the <code>HF_TOKEN</code> environment variable or passing the <code>storage_options</code> parameter to helper functions below:",jl,N,ol,S,Kl='For more details about authentication, check out <a href="https://huggingface.co/docs/huggingface_hub/quick-start#authentication" rel="nofollow">this guide</a>.',ml,A,ul,Y,Pl="PySpark doesn’t have an official support for Hugging Face paths, so we provide a helper function to read datasets in a distributed manner.",dl,X,ls="For example you can read Parquet files from Hugging Face in an optimized way using PyArrow by defining this <code>read_parquet</code> helper function:",Il,Q,bl,F,ss=`Here is how we can use this on the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset.
It is a gated repository, users have to accept the terms of use before accessing it.`,Cl,j,as='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-dark-min.png"/>',Bl,W,ns="We use the <code>read_parquet</code> function to read data from the dataset, compute the number of dialogue per language and filter the dataset.",gl,V,ts="After logging-in to access the gated repository, we can run:",Gl,E,fl,z,es=`To compute the number of dialogues per language we run this code.
The <code>columns</code> argument is useful to only load the data we need, since PySpark doesn’t enable predicate push-down in this case.
There is also a <code>filters</code> argument to only load data with values within a certain range.`,kl,v,Zl,_,Ms="To filter the dataset and only keep dialogues in Chinese:",Rl,q,Nl,H,Sl,D,ps="We also provide a helper function to write datasets in a distributed manner to a Hugging Face repository.",Al,x,ys=`You can write a PySpark Dataframe to Hugging Face using this <code>write_parquet</code> helper functions based on the <code>huggingface_hub</code> API.
In particular it uses the <code>preupload_lfs_files</code> utility to upload Parquet files in parallel in a distributed manner, and only commit the files once they’re all uploaded:`,Yl,O,Xl,L,Js='Here is how we can use this function to write the filtered version of the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset back to Hugging Face.',Ql,$,Us=`First you need to <a href="https://huggingface.co/new-dataset" rel="nofollow">create a dataset repository</a>, e.g. <code>username/Infinity-Instruct-Chinese-Only</code> (you can set it to private if you want).
Then, make sure you are authenticated and you can run:`,Fl,K,Wl,o,is='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-dark-min.png"/>',Vl,P,El,sl,zl;return m=new al({props:{title:"Spark",local:"spark",headingTag:"h1"}}),I=new al({props:{title:"Installation",local:"installation",headingTag:"h2"}}),C=new w({props:{code:"cGlwJTIwaW5zdGFsbCUyMGh1Z2dpbmdmYWNlX2h1Yg==",highlighted:'pip <span class="hljs-keyword">install</span> huggingface_hub',wrap:!1}}),g=new w({props:{code:"cGlwJTIwaW5zdGFsbCUyMHB5YXJyb3c=",highlighted:'pip <span class="hljs-keyword">install</span> pyarrow',wrap:!1}}),G=new al({props:{title:"Authentication",local:"authentication",headingTag:"h2"}}),Z=new w({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),N=new w({props:{code:"c3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIydG9rZW4lMjIlM0ElMjAlMjJoZl94eHglMjIlN0Q=",highlighted:'storage_options = {<span class="hljs-string">&quot;token&quot;</span>: <span class="hljs-string">&quot;hf_xxx&quot;</span>}',wrap:!1}}),A=new al({props:{title:"Read",local:"read",headingTag:"h2"}}),Q=new w({props:{code:" ",highlighted:`<span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Iterator, <span class="hljs-type">Optional</span>, <span class="hljs-type">Union</span>

<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFileSystem
<span class="hljs-keyword">from</span> pyspark.sql.dataframe <span class="hljs-keyword">import</span> DataFrame
<span class="hljs-keyword">from</span> pyspark.sql.pandas.types <span class="hljs-keyword">import</span> from_arrow_schema


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_read</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], columns: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">str</span>]], filters: <span class="hljs-type">Optional</span>[<span class="hljs-type">Union</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>], <span class="hljs-built_in">list</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>]]]], **kwargs</span>) -&gt; Iterator[pa.RecordBatch]:
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> iterator:
        paths = batch[<span class="hljs-number">0</span>].to_pylist()
        ds = pq.ParquetDataset(paths, **kwargs)
        <span class="hljs-keyword">yield</span> <span class="hljs-keyword">from</span> ds._dataset.to_batches(columns=columns, <span class="hljs-built_in">filter</span>=pq.filters_to_expression(filters) <span class="hljs-keyword">if</span> filters <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">read_parquet</span>(<span class="hljs-params">
    path: <span class="hljs-built_in">str</span>,
    columns: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">str</span>]] = <span class="hljs-literal">None</span>,
    filters: <span class="hljs-type">Optional</span>[<span class="hljs-type">Union</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>], <span class="hljs-built_in">list</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>]]]] = <span class="hljs-literal">None</span>,
    **kwargs,
</span>) -&gt; DataFrame:
    <span class="hljs-string">&quot;&quot;&quot;
    Loads Parquet files from Hugging Face using PyArrow, returning a PySPark \`DataFrame\`.

    It reads Parquet files in a distributed manner.

    Access private or gated repositories using \`huggingface-cli login\` or passing a token
    using the \`storage_options\` argument: \`storage_options={&quot;token&quot;: &quot;hf_xxx&quot;}\`

    Parameters
    ----------
    path : str
        Path to the file. Prefix with a protocol like \`hf://\` to read from Hugging Face.
        You can read from multiple files if you pass a globstring.
    columns : list, default None
        If not None, only these columns will be read from the file.
    filters : List[Tuple] or List[List[Tuple]], default None
        To filter out data.
        Filter syntax: [[(column, op, val), ...],...]
        where op is [==, =, &gt;, &gt;=, &lt;, &lt;=, !=, in, not in]
        The innermost tuples are transposed into a set of filters applied
        through an \`AND\` operation.
        The outer list combines these sets of filters through an \`OR\`
        operation.
        A single list of tuples can also be used, meaning that no \`OR\`
        operation between set of filters is to be conducted.

    **kwargs
        Any additional kwargs are passed to pyarrow.parquet.ParquetDataset.

    Returns
    -------
    DataFrame
        DataFrame based on parquet file.

    Examples
    --------
    &gt;&gt;&gt; path = &quot;hf://datasets/username/dataset/data.parquet&quot;
    &gt;&gt;&gt; pd.DataFrame({&quot;foo&quot;: range(5), &quot;bar&quot;: range(5, 10)}).to_parquet(path)
    &gt;&gt;&gt; read_parquet(path).show()
    +---+---+
    |foo|bar|
    +---+---+
    |  0|  5|
    |  1|  6|
    |  2|  7|
    |  3|  8|
    |  4|  9|
    +---+---+
    &gt;&gt;&gt; read_parquet(path, columns=[&quot;bar&quot;]).show()
    +---+
    |bar|
    +---+
    |  5|
    |  6|
    |  7|
    |  8|
    |  9|
    +---+
    &gt;&gt;&gt; sel = [(&quot;foo&quot;, &quot;&gt;&quot;, 2)]
    &gt;&gt;&gt; read_parquet(path, filters=sel).show()
    +---+---+
    |foo|bar|
    +---+---+
    |  3|  8|
    |  4|  9|
    +---+---+
    &quot;&quot;&quot;</span>
    filesystem: HfFileSystem = kwargs.pop(<span class="hljs-string">&quot;filesystem&quot;</span>) <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;filesystem&quot;</span> <span class="hljs-keyword">in</span> kwargs <span class="hljs-keyword">else</span> HfFileSystem(**kwargs.pop(<span class="hljs-string">&quot;storage_options&quot;</span>, {}))
    paths = filesystem.glob(path)
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> paths:
        <span class="hljs-keyword">raise</span> FileNotFoundError(<span class="hljs-string">f&quot;Counldn&#x27;t find any file at <span class="hljs-subst">{path}</span>&quot;</span>)
    rdd = spark.sparkContext.parallelize([{<span class="hljs-string">&quot;path&quot;</span>: path} <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> paths], <span class="hljs-built_in">len</span>(paths))
    df = spark.createDataFrame(rdd)
    arrow_schema = pq.read_schema(filesystem.<span class="hljs-built_in">open</span>(paths[<span class="hljs-number">0</span>]))
    schema = pa.schema([field <span class="hljs-keyword">for</span> field <span class="hljs-keyword">in</span> arrow_schema <span class="hljs-keyword">if</span> (columns <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> field.name <span class="hljs-keyword">in</span> columns)], metadata=arrow_schema.metadata)
    <span class="hljs-keyword">return</span> df.mapInArrow(
        partial(_read, columns=columns, filters=filters, filesystem=filesystem, schema=arrow_schema, **kwargs),
        from_arrow_schema(schema),
    )`,wrap:!1}}),E=new w({props:{code:"ZnJvbSUyMHB5c3Bhcmsuc3FsJTIwaW1wb3J0JTIwU3BhcmtTZXNzaW9uJTBBc3BhcmslMjAlM0QlMjBTcGFya1Nlc3Npb24uYnVpbGRlci5hcHBOYW1lKCUyMmRlbW8lMjIpLmdldE9yQ3JlYXRlKCklMEFkZiUyMCUzRCUyMHJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGQkFBSSUyRkluZmluaXR5LUluc3RydWN0JTJGN00lMkYqLnBhcnF1ZXQlMjIpJTBBZGYuc2hvdygp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
<span class="hljs-meta">&gt;&gt;&gt; </span>spark = SparkSession.builder.appName(<span class="hljs-string">&quot;demo&quot;</span>).getOrCreate()
<span class="hljs-meta">&gt;&gt;&gt; </span>df = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df.show()
+---+----------------------------+-----+----------+--------------------+        
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|              source|
+---+----------------------------+-----+----------+--------------------+
|  <span class="hljs-number">0</span>|        [{human, <span class="hljs-keyword">def</span> <span class="hljs-title function_">exti</span>...|     |        en|      code_exercises|
|  <span class="hljs-number">1</span>|        [{human, See the ...|     |        en|                flan|
|  <span class="hljs-number">2</span>|        [{human, This <span class="hljs-keyword">is</span> ...|     |        en|                flan|
|  <span class="hljs-number">3</span>|        [{human, If you d...|     |        en|                flan|
|  <span class="hljs-number">4</span>|        [{human, In a Uni...|     |        en|                flan|
|  <span class="hljs-number">5</span>|        [{human, Read the...|     |        en|                flan|
|  <span class="hljs-number">6</span>|        [{human, You are ...|     |        en|          code_bagel|
|  <span class="hljs-number">7</span>|        [{human, I want y...|     |        en|          Subjective|
|  <span class="hljs-number">8</span>|        [{human, Given th...|     |        en|                flan|
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|          Subjective|
| <span class="hljs-number">10</span>|        [{human, Provide ...|     |        en|self-oss-instruct...|
| <span class="hljs-number">11</span>|        [{human, The univ...|     |        en|                flan|
| <span class="hljs-number">12</span>|        [{human, Q: I am ...|     |        en|                flan|
| <span class="hljs-number">13</span>|        [{human, What <span class="hljs-keyword">is</span> ...|     |        en|      OpenHermes-<span class="hljs-number">2.5</span>|
| <span class="hljs-number">14</span>|        [{human, In react...|     |        en|                flan|
| <span class="hljs-number">15</span>|        [{human, Write Py...|     |        en|      code_exercises|
| <span class="hljs-number">16</span>|        [{human, Find the...|     |        en|            MetaMath|
| <span class="hljs-number">17</span>|        [{human, Three of...|     |        en|            MetaMath|
| <span class="hljs-number">18</span>|        [{human, Chandra ...|     |        en|            MetaMath|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|          Subjective|
+---+----------------------------+-----+----------+--------------------+`,wrap:!1}}),v=new w({props:{code:"ZGZfbGFuZ2RldGVjdF9vbmx5JTIwJTNEJTIwcmVhZF9wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZCQUFJJTJGSW5maW5pdHktSW5zdHJ1Y3QlMkY3TSUyRioucGFycXVldCUyMiUyQyUyMGNvbHVtbnMlM0QlNUIlMjJsYW5nZGV0ZWN0JTIyJTVEKSUwQWRmX2xhbmdkZXRlY3Rfb25seS5ncm91cEJ5KCUyMmxhbmdkZXRlY3QlMjIpLmNvdW50KCkuc2hvdygp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>, columns=[<span class="hljs-string">&quot;langdetect&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only.groupBy(<span class="hljs-string">&quot;langdetect&quot;</span>).count().show()
+----------+-------+                                                            
|langdetect|  count|
+----------+-------+
|        en|<span class="hljs-number">6697793</span>|
|     zh-cn| <span class="hljs-number">751313</span>|
+----------+-------+`,wrap:!1}}),q=new w({props:{code:"Y3JpdGVyaWElMjAlM0QlMjAlNUIoJTIybGFuZ2RldGVjdCUyMiUyQyUyMCUyMiUzRCUyMiUyQyUyMCUyMnpoLWNuJTIyKSU1RCUwQWRmX2NoaW5lc2Vfb25seSUyMCUzRCUyMHJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGQkFBSSUyRkluZmluaXR5LUluc3RydWN0JTJGN00lMkYqLnBhcnF1ZXQlMjIlMkMlMjBmaWx0ZXJzJTNEY3JpdGVyaWEpJTBBZGZfY2hpbmVzZV9vbmx5",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>criteria = [(<span class="hljs-string">&quot;langdetect&quot;</span>, <span class="hljs-string">&quot;=&quot;</span>, <span class="hljs-string">&quot;zh-cn&quot;</span>)]
<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>, filters=criteria)
<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only
+---+----------------------------+-----+----------+----------+                  
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|    source|
+---+----------------------------+-----+----------+----------+
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|Subjective|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|Subjective|
| <span class="hljs-number">38</span>| [{human, 某个考试共有A、...|     |     zh-cn|Subjective|
| <span class="hljs-number">39</span>|[{human, 撰写一篇关于斐波...|     |     zh-cn|Subjective|
| <span class="hljs-number">57</span>|[{human, 总结世界历史上的...|     |     zh-cn|Subjective|
| <span class="hljs-number">61</span>|[{human, 生成一则广告词。...|     |     zh-cn|Subjective|
| <span class="hljs-number">66</span>|[{human, 描述一个有效的团...|     |     zh-cn|Subjective|
| <span class="hljs-number">94</span>|[{human, 如果比利和蒂芙尼...|     |     zh-cn|Subjective|
|<span class="hljs-number">102</span>|[{human, 生成一句英文名言...|     |     zh-cn|Subjective|
|<span class="hljs-number">106</span>|[{human, 写一封感谢信，感...|     |     zh-cn|Subjective|
|<span class="hljs-number">118</span>| [{human, 生成一个故事。}...|     |     zh-cn|Subjective|
|<span class="hljs-number">174</span>|[{human, 高胆固醇水平的后...|     |     zh-cn|Subjective|
|<span class="hljs-number">180</span>|[{human, 基于以下角色信息...|     |     zh-cn|Subjective|
|<span class="hljs-number">192</span>|[{human, 请写一篇文章，概...|     |     zh-cn|Subjective|
|<span class="hljs-number">221</span>|[{human, 以诗歌形式表达对...|     |     zh-cn|Subjective|
|<span class="hljs-number">228</span>|[{human, 根据给定的指令，...|     |     zh-cn|Subjective|
|<span class="hljs-number">236</span>|[{human, 打开一个新的生成...|     |     zh-cn|Subjective|
|<span class="hljs-number">260</span>|[{human, 生成一个有关未来...|     |     zh-cn|Subjective|
|<span class="hljs-number">268</span>|[{human, 如果有一定数量的...|     |     zh-cn|Subjective|
|<span class="hljs-number">273</span>| [{human, 题目：小明有<span class="hljs-number">5</span>个...|     |     zh-cn|Subjective|
+---+----------------------------+-----+----------+----------+`,wrap:!1}}),H=new al({props:{title:"Write",local:"write",headingTag:"h2"}}),O=new w({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> math
<span class="hljs-keyword">import</span> pickle
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Iterator, <span class="hljs-type">Optional</span>

<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> CommitOperationAdd, HfFileSystem
<span class="hljs-keyword">from</span> pyspark.sql.dataframe <span class="hljs-keyword">import</span> DataFrame
<span class="hljs-keyword">from</span> pyspark.sql.pandas.types <span class="hljs-keyword">import</span> from_arrow_schema, to_arrow_schema


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_preupload</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], path: <span class="hljs-built_in">str</span>, schema: pa.Schema, filesystem: HfFileSystem, row_group_size: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">int</span>] = <span class="hljs-literal">None</span>, **kwargs</span>) -&gt; Iterator[pa.RecordBatch]:
    resolved_path = filesystem.resolve_path(path)
    <span class="hljs-keyword">with</span> tempfile.NamedTemporaryFile(suffix=<span class="hljs-string">&quot;.parquet&quot;</span>) <span class="hljs-keyword">as</span> temp_file:
        <span class="hljs-keyword">with</span> pq.ParquetWriter(temp_file.name, schema=schema, **kwargs) <span class="hljs-keyword">as</span> writer:
            <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> iterator:
                writer.write_batch(batch, row_group_size=row_group_size)
        addition = CommitOperationAdd(path_in_repo=temp_file.name, path_or_fileobj=temp_file.name)
        filesystem._api.preupload_lfs_files(repo_id=resolved_path.repo_id, additions=[addition], repo_type=resolved_path.repo_type, revision=resolved_path.revision)
    <span class="hljs-keyword">yield</span> pa.record_batch({<span class="hljs-string">&quot;addition&quot;</span>: [pickle.dumps(addition)]}, schema=pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()}))


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_commit</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], path: <span class="hljs-built_in">str</span>, filesystem: HfFileSystem, max_operations_per_commit=<span class="hljs-number">50</span></span>) -&gt; Iterator[pa.RecordBatch]:
    resolved_path = filesystem.resolve_path(path)
    additions: <span class="hljs-built_in">list</span>[CommitOperationAdd] = [pickle.loads(addition) <span class="hljs-keyword">for</span> addition <span class="hljs-keyword">in</span> pa.Table.from_batches(iterator, schema=pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()}))[<span class="hljs-number">0</span>].to_pylist()]
    num_commits = math.ceil(<span class="hljs-built_in">len</span>(additions) / max_operations_per_commit)
    <span class="hljs-keyword">for</span> shard_idx, addition <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(additions):
        addition.path_in_repo = resolved_path.path_in_repo.replace(<span class="hljs-string">&quot;{shard_idx:05d}&quot;</span>, <span class="hljs-string">f&quot;<span class="hljs-subst">{shard_idx:05d}</span>&quot;</span>)
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, num_commits):
        operations = additions[i * max_operations_per_commit : (i + <span class="hljs-number">1</span>) * max_operations_per_commit]
        commit_message = <span class="hljs-string">&quot;Upload using PySpark&quot;</span> + (<span class="hljs-string">f&quot; (part <span class="hljs-subst">{i:05d}</span>-of-<span class="hljs-subst">{num_commits:05d}</span>)&quot;</span> <span class="hljs-keyword">if</span> num_commits &gt; <span class="hljs-number">1</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;&quot;</span>)
        filesystem._api.create_commit(repo_id=resolved_path.repo_id, repo_type=resolved_path.repo_type, revision=resolved_path.revision, operations=operations, commit_message=commit_message)
        <span class="hljs-keyword">yield</span> pa.record_batch({<span class="hljs-string">&quot;path&quot;</span>: [addition.path_in_repo <span class="hljs-keyword">for</span> addition <span class="hljs-keyword">in</span> operations]}, schema=pa.schema({<span class="hljs-string">&quot;path&quot;</span>: pa.string()}))


<span class="hljs-keyword">def</span> <span class="hljs-title function_">write_parquet</span>(<span class="hljs-params">df: DataFrame, path: <span class="hljs-built_in">str</span>, **kwargs</span>) -&gt; <span class="hljs-literal">None</span>:
    <span class="hljs-string">&quot;&quot;&quot;
    Write Parquet files to Hugging Face using PyArrow.

    It uploads Parquet files in a distributed manner in two steps:

    1. Preupload the Parquet files in parallel in a distributed banner
    2. Commit the preuploaded files

    Authenticate using \`huggingface-cli login\` or passing a token
    using the \`storage_options\` argument: \`storage_options={&quot;token&quot;: &quot;hf_xxx&quot;}\`

    Parameters
    ----------
    path : str
        Path of the file or directory. Prefix with a protocol like \`hf://\` to read from Hugging Face.
        It writes Parquet files in the form &quot;part-xxxxx.parquet&quot;, or to a single file if \`path ends with &quot;.parquet&quot;.

    **kwargs
        Any additional kwargs are passed to pyarrow.parquet.ParquetWriter.

    Returns
    -------
    DataFrame
        DataFrame based on parquet file.

    Examples
    --------
    &gt;&gt;&gt; spark.createDataFrame(pd.DataFrame({&quot;foo&quot;: range(5), &quot;bar&quot;: range(5, 10)}))
    &gt;&gt;&gt; # Save to one file
    &gt;&gt;&gt; write_parquet(df, &quot;hf://datasets/username/dataset/data.parquet&quot;)
    &gt;&gt;&gt; # OR save to a directory (possibly in many files)
    &gt;&gt;&gt; write_parquet(df, &quot;hf://datasets/username/dataset&quot;)
    &quot;&quot;&quot;</span>
    filesystem: HfFileSystem = kwargs.pop(<span class="hljs-string">&quot;filesystem&quot;</span>, HfFileSystem(**kwargs.pop(<span class="hljs-string">&quot;storage_options&quot;</span>, {})))
    <span class="hljs-keyword">if</span> path.endswith(<span class="hljs-string">&quot;.parquet&quot;</span>) <span class="hljs-keyword">or</span> path.endswith(<span class="hljs-string">&quot;.pq&quot;</span>):
        df = df.coalesce(<span class="hljs-number">1</span>)
    <span class="hljs-keyword">else</span>:
        path += <span class="hljs-string">&quot;/part-{shard_idx:05d}.parquet&quot;</span>
    df.mapInArrow(
        partial(_preupload, path=path, schema=to_arrow_schema(df.schema), filesystem=filesystem, **kwargs),
        from_arrow_schema(pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()})),
    ).coalesce(<span class="hljs-number">1</span>).mapInArrow(
        partial(_commit, path=path, filesystem=filesystem),
        from_arrow_schema(pa.schema({<span class="hljs-string">&quot;path&quot;</span>: pa.string()})),
    ).collect()`,wrap:!1}}),K=new w({props:{code:"d3JpdGVfcGFycXVldChkZl9jaGluZXNlX29ubHklMkMlMjAlMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZJbmZpbml0eS1JbnN0cnVjdC1DaGluZXNlLU9ubHklMjIpJTBBdG1waDlqd3U5cHkucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjVNJTJGNTAuNU0lMjAlNUIwMCUzQTAzJTNDMDAlM0EwMCUyQyUyMDE0LjZNQiUyRnMlNUQlMEF0bXAwb3F0OTluYy5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOE0lMkY1MC44TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMTcuOU1CJTJGcyU1RCUwQXRtcGduaXprd3FwLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC41TSUyRjUwLjVNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAxOS42TUIlMkZzJTVEJTBBdG1wYW5tMDRrNG4ucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUxLjRNJTJGNTEuNE0lMjAlNUIwMCUzQTAyJTNDMDAlM0EwMCUyQyUyMDIyLjlNQiUyRnMlNUQlMEF0bXAxNHV5OW9xYi5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuNE0lMkY1MC40TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMjMuME1CJTJGcyU1RCUwQXRtcGNwOHRfcWRsLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC40TSUyRjUwLjRNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy41TUIlMkZzJTVEJTBBdG1wanVpNW1uczgucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjNNJTJGNTAuM00lMjAlNUIwMCUzQTAyJTNDMDAlM0EwMCUyQyUyMDI0LjFNQiUyRnMlNUQlMEF0bXB5ZHFoNm9kMS5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOU0lMkY1MC45TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMjMuOE1CJTJGcyU1RCUwQXRtcDUyZjJ0OHR1LnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC41TSUyRjUwLjVNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy43TUIlMkZzJTVEJTBBdG1wZzdlZ3YzeWUucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjFNJTJGNTAuMU0lMjAlNUIwMCUzQTA2JTNDMDAlM0EwMCUyQyUyMDcuNjhNQiUyRnMlNUQlMEF0bXAyczBmcTJobS5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOE0lMkY1MC44TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMTguMU1CJTJGcyU1RCUwQXRtcG1qOTdhYjMwLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA3MS4zTSUyRjcxLjNNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy45TUIlMkZzJTVE",highlighted:`write_parquet(df_chinese_only, <span class="hljs-string">&quot;hf://datasets/username/Infinity-Instruct-Chinese-Only&quot;</span>)
tmph9jwu9py.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:03&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">14.6</span>MB/s]
tmp0oqt99nc.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.8</span>M/<span class="hljs-number">50.8</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">17.9</span>MB/s]
tmpgnizkwqp.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">19.6</span>MB/s]
tmpanm04k4n.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">51.4</span>M/<span class="hljs-number">51.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">22.9</span>MB/s]
tmp14uy9oqb.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.4</span>M/<span class="hljs-number">50.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.0</span>MB/s]
tmpcp8t_qdl.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.4</span>M/<span class="hljs-number">50.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.5</span>MB/s]
tmpjui5mns8.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.3</span>M/<span class="hljs-number">50.3</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">24.1</span>MB/s]
tmpydqh6od1.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.9</span>M/<span class="hljs-number">50.9</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.8</span>MB/s]
tmp52f2t8tu.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.7</span>MB/s]
tmpg7egv3ye.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.1</span>M/<span class="hljs-number">50.1</span>M [<span class="hljs-number">00</span>:06&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">7.68</span>MB/s]
tmp2s0fq2hm.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.8</span>M/<span class="hljs-number">50.8</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">18.1</span>MB/s]
tmpmj97ab30.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">71.3</span>M/<span class="hljs-number">71.3</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.9</span>MB/s]`,wrap:!1}}),P=new us({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-spark.md"}}),{c(){h=M("meta"),nl=t(),ll=M("p"),tl=t(),J(m.$$.fragment),el=t(),u=M("p"),u.textContent=ql,Ml=t(),d=M("p"),d.innerHTML=Hl,pl=t(),J(I.$$.fragment),yl=t(),b=M("p"),b.innerHTML=Dl,Jl=t(),J(C.$$.fragment),Ul=t(),B=M("p"),B.innerHTML=xl,il=t(),J(g.$$.fragment),cl=t(),J(G.$$.fragment),Tl=t(),f=M("p"),f.textContent=Ol,rl=t(),k=M("p"),k.textContent=Ll,wl=t(),J(Z.$$.fragment),hl=t(),R=M("p"),R.innerHTML=$l,jl=t(),J(N.$$.fragment),ol=t(),S=M("p"),S.innerHTML=Kl,ml=t(),J(A.$$.fragment),ul=t(),Y=M("p"),Y.textContent=Pl,dl=t(),X=M("p"),X.innerHTML=ls,Il=t(),J(Q.$$.fragment),bl=t(),F=M("p"),F.innerHTML=ss,Cl=t(),j=M("div"),j.innerHTML=as,Bl=t(),W=M("p"),W.innerHTML=ns,gl=t(),V=M("p"),V.textContent=ts,Gl=t(),J(E.$$.fragment),fl=t(),z=M("p"),z.innerHTML=es,kl=t(),J(v.$$.fragment),Zl=t(),_=M("p"),_.textContent=Ms,Rl=t(),J(q.$$.fragment),Nl=t(),J(H.$$.fragment),Sl=t(),D=M("p"),D.textContent=ps,Al=t(),x=M("p"),x.innerHTML=ys,Yl=t(),J(O.$$.fragment),Xl=t(),L=M("p"),L.innerHTML=Js,Ql=t(),$=M("p"),$.innerHTML=Us,Fl=t(),J(K.$$.fragment),Wl=t(),o=M("div"),o.innerHTML=is,Vl=t(),J(P.$$.fragment),El=t(),sl=M("p"),this.h()},l(l){const s=os("svelte-u9bgzb",document.head);h=p(s,"META",{name:!0,content:!0}),s.forEach(a),nl=e(l),ll=p(l,"P",{}),cs(ll).forEach(a),tl=e(l),U(m.$$.fragment,l),el=e(l),u=p(l,"P",{"data-svelte-h":!0}),y(u)!=="svelte-s0r53o"&&(u.textContent=ql),Ml=e(l),d=p(l,"P",{"data-svelte-h":!0}),y(d)!=="svelte-12z8vci"&&(d.innerHTML=Hl),pl=e(l),U(I.$$.fragment,l),yl=e(l),b=p(l,"P",{"data-svelte-h":!0}),y(b)!=="svelte-13h0lcm"&&(b.innerHTML=Dl),Jl=e(l),U(C.$$.fragment,l),Ul=e(l),B=p(l,"P",{"data-svelte-h":!0}),y(B)!=="svelte-ml7hwl"&&(B.innerHTML=xl),il=e(l),U(g.$$.fragment,l),cl=e(l),U(G.$$.fragment,l),Tl=e(l),f=p(l,"P",{"data-svelte-h":!0}),y(f)!=="svelte-aqw7kt"&&(f.textContent=Ol),rl=e(l),k=p(l,"P",{"data-svelte-h":!0}),y(k)!=="svelte-pdivv8"&&(k.textContent=Ll),wl=e(l),U(Z.$$.fragment,l),hl=e(l),R=p(l,"P",{"data-svelte-h":!0}),y(R)!=="svelte-1pzllua"&&(R.innerHTML=$l),jl=e(l),U(N.$$.fragment,l),ol=e(l),S=p(l,"P",{"data-svelte-h":!0}),y(S)!=="svelte-ygg3x4"&&(S.innerHTML=Kl),ml=e(l),U(A.$$.fragment,l),ul=e(l),Y=p(l,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-7gpw7a"&&(Y.textContent=Pl),dl=e(l),X=p(l,"P",{"data-svelte-h":!0}),y(X)!=="svelte-4fzrqm"&&(X.innerHTML=ls),Il=e(l),U(Q.$$.fragment,l),bl=e(l),F=p(l,"P",{"data-svelte-h":!0}),y(F)!=="svelte-1vx9tic"&&(F.innerHTML=ss),Cl=e(l),j=p(l,"DIV",{class:!0,"data-svelte-h":!0}),y(j)!=="svelte-1ct11n9"&&(j.innerHTML=as),Bl=e(l),W=p(l,"P",{"data-svelte-h":!0}),y(W)!=="svelte-1p4gwvr"&&(W.innerHTML=ns),gl=e(l),V=p(l,"P",{"data-svelte-h":!0}),y(V)!=="svelte-19d7o0j"&&(V.textContent=ts),Gl=e(l),U(E.$$.fragment,l),fl=e(l),z=p(l,"P",{"data-svelte-h":!0}),y(z)!=="svelte-q1u30o"&&(z.innerHTML=es),kl=e(l),U(v.$$.fragment,l),Zl=e(l),_=p(l,"P",{"data-svelte-h":!0}),y(_)!=="svelte-e2gcj5"&&(_.textContent=Ms),Rl=e(l),U(q.$$.fragment,l),Nl=e(l),U(H.$$.fragment,l),Sl=e(l),D=p(l,"P",{"data-svelte-h":!0}),y(D)!=="svelte-nrb4bc"&&(D.textContent=ps),Al=e(l),x=p(l,"P",{"data-svelte-h":!0}),y(x)!=="svelte-168mt1p"&&(x.innerHTML=ys),Yl=e(l),U(O.$$.fragment,l),Xl=e(l),L=p(l,"P",{"data-svelte-h":!0}),y(L)!=="svelte-1d35p3o"&&(L.innerHTML=Js),Ql=e(l),$=p(l,"P",{"data-svelte-h":!0}),y($)!=="svelte-wkzo7"&&($.innerHTML=Us),Fl=e(l),U(K.$$.fragment,l),Wl=e(l),o=p(l,"DIV",{class:!0,"data-svelte-h":!0}),y(o)!=="svelte-tmflad"&&(o.innerHTML=is),Vl=e(l),U(P.$$.fragment,l),El=e(l),sl=p(l,"P",{}),cs(sl).forEach(a),this.h()},h(){vl(h,"name","hf:doc:metadata"),vl(h,"content",Is),vl(j,"class","flex justify-center"),vl(o,"class","flex justify-center")},m(l,s){ms(document.head,h),n(l,nl,s),n(l,ll,s),n(l,tl,s),i(m,l,s),n(l,el,s),n(l,u,s),n(l,Ml,s),n(l,d,s),n(l,pl,s),i(I,l,s),n(l,yl,s),n(l,b,s),n(l,Jl,s),i(C,l,s),n(l,Ul,s),n(l,B,s),n(l,il,s),i(g,l,s),n(l,cl,s),i(G,l,s),n(l,Tl,s),n(l,f,s),n(l,rl,s),n(l,k,s),n(l,wl,s),i(Z,l,s),n(l,hl,s),n(l,R,s),n(l,jl,s),i(N,l,s),n(l,ol,s),n(l,S,s),n(l,ml,s),i(A,l,s),n(l,ul,s),n(l,Y,s),n(l,dl,s),n(l,X,s),n(l,Il,s),i(Q,l,s),n(l,bl,s),n(l,F,s),n(l,Cl,s),n(l,j,s),n(l,Bl,s),n(l,W,s),n(l,gl,s),n(l,V,s),n(l,Gl,s),i(E,l,s),n(l,fl,s),n(l,z,s),n(l,kl,s),i(v,l,s),n(l,Zl,s),n(l,_,s),n(l,Rl,s),i(q,l,s),n(l,Nl,s),i(H,l,s),n(l,Sl,s),n(l,D,s),n(l,Al,s),n(l,x,s),n(l,Yl,s),i(O,l,s),n(l,Xl,s),n(l,L,s),n(l,Ql,s),n(l,$,s),n(l,Fl,s),i(K,l,s),n(l,Wl,s),n(l,o,s),n(l,Vl,s),i(P,l,s),n(l,El,s),n(l,sl,s),zl=!0},p:rs,i(l){zl||(c(m.$$.fragment,l),c(I.$$.fragment,l),c(C.$$.fragment,l),c(g.$$.fragment,l),c(G.$$.fragment,l),c(Z.$$.fragment,l),c(N.$$.fragment,l),c(A.$$.fragment,l),c(Q.$$.fragment,l),c(E.$$.fragment,l),c(v.$$.fragment,l),c(q.$$.fragment,l),c(H.$$.fragment,l),c(O.$$.fragment,l),c(K.$$.fragment,l),c(P.$$.fragment,l),zl=!0)},o(l){T(m.$$.fragment,l),T(I.$$.fragment,l),T(C.$$.fragment,l),T(g.$$.fragment,l),T(G.$$.fragment,l),T(Z.$$.fragment,l),T(N.$$.fragment,l),T(A.$$.fragment,l),T(Q.$$.fragment,l),T(E.$$.fragment,l),T(v.$$.fragment,l),T(q.$$.fragment,l),T(H.$$.fragment,l),T(O.$$.fragment,l),T(K.$$.fragment,l),T(P.$$.fragment,l),zl=!1},d(l){l&&(a(nl),a(ll),a(tl),a(el),a(u),a(Ml),a(d),a(pl),a(yl),a(b),a(Jl),a(Ul),a(B),a(il),a(cl),a(Tl),a(f),a(rl),a(k),a(wl),a(hl),a(R),a(jl),a(ol),a(S),a(ml),a(ul),a(Y),a(dl),a(X),a(Il),a(bl),a(F),a(Cl),a(j),a(Bl),a(W),a(gl),a(V),a(Gl),a(fl),a(z),a(kl),a(Zl),a(_),a(Rl),a(Nl),a(Sl),a(D),a(Al),a(x),a(Yl),a(Xl),a(L),a(Ql),a($),a(Fl),a(Wl),a(o),a(Vl),a(El),a(sl)),a(h),r(m,l),r(I,l),r(C,l),r(g,l),r(G,l),r(Z,l),r(N,l),r(A,l),r(Q,l),r(E,l),r(v,l),r(q,l),r(H,l),r(O,l),r(K,l),r(P,l)}}}const Is='{"title":"Spark","local":"spark","sections":[{"title":"Installation","local":"installation","sections":[],"depth":2},{"title":"Authentication","local":"authentication","sections":[],"depth":2},{"title":"Read","local":"read","sections":[],"depth":2},{"title":"Write","local":"write","sections":[],"depth":2}],"depth":1}';function bs(_l){return ws(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fs extends hs{constructor(h){super(),js(this,h,bs,ds,Ts,{})}}export{fs as component};
