import{s as ds,f as $t,o as js,n as ms}from"../chunks/scheduler.65852ee5.js";import{S as Js,i as hs,g as M,s as a,r,A as us,h as o,f as e,c as n,j as ys,u as p,x as i,k as u,y as Us,a as s,v as c,d as y,t as d,w as j}from"../chunks/index.aa74147d.js";import{T as ws}from"../chunks/Tip.bb8ccac8.js";import{D as Ts,C as J}from"../chunks/DocNotebookDropdown.479f4286.js";import{H as h,E as Is}from"../chunks/EditOnGithub.4eda6a96.js";function bs(qt){let m,b="The code below will be updated once the Inference API recipe is finished.";return{c(){m=M("p"),m.textContent=b},l(U){m=o(U,"P",{"data-svelte-h":!0}),i(m)!=="svelte-1b5x1p8"&&(m.textContent=b)},m(U,xt){s(U,m,xt)},p:ms,d(U){U&&e(m)}}}function fs(qt){let m,b,U,xt,f,Lt,g,Dt,C,we='<em>Authored by: <a href="https://huggingface.co/MoritzLaurer" rel="nofollow">Moritz Laurer</a></em>',Pt,G,Te="This notebook illustrates the workflow for systematically evaluating LLM outputs and creating LLM training data. You can start by using this notebook for evaluating the zeroshot performance of your favourite LLM on your task without any fine-tuning. If you want to improve performance, you can then easily reuse this workflow to create training data.",Kt,B,Ie="<strong>Example use-case: code generation.</strong> For this tutorial we demonstrate how to create high quality test &amp; train data for <em>code generation tasks</em>. The same workflow can, however, be adapted to any other task that’s relevant for your specific use-case.",Ot,Z,be="<strong>In this notebook, we:</strong>",tl,A,fe="<li>Download data for the example task.</li> <li>Prompt two LLMs to respond to these tasks. This results in “synthetic data” to speed up manual data creation.</li> <li>Create an Argilla annotation interface on HF Spaces to compare and evaluate the outputs from the two LLMs.</li> <li>Upload the example data and the zeroshot LLM responses into the Argilla annotation interface.</li> <li>Download the annotated data.</li>",ll,R,ge="You can adapt this notebook to your needs, e.g. by using a different LLM and API provider for step (2) or by adapting the annotation interface in step (3).",el,k,sl,W,al,v,nl,N,Ml,X,Ce='We first download an example dataset that contains code generation tasks for LLMs. We want to evaluate how well two different LLMs perform on these code generation tasks. We use instructions from the <a href="https://huggingface.co/datasets/bigcode/self-oss-instruct-sc2-exec-filter-50k" rel="nofollow">bigcode/self-oss-instruct-sc2-exec-filter-50k</a> dataset that was used to train the <a href="https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1" rel="nofollow">StarCoder2-Instruct</a> model.',ol,Q,il,Y,Ge="Dataset structure:\n Dataset(&#123;\n    features: ['fingerprint', 'sha1', 'seed', 'response', 'concepts', 'prompt', 'instruction', 'id'],\n    num_rows: 3\n}) \n\nExample instructions:\n ['Write a Python function named `get_value` that takes a matrix (represented by a list of lists) and a tuple of indices, and returns the value at that index in the matrix. The function should handle index out of range errors by returning None.', 'Write a Python function `check_collision` that takes a list of `rectangles` as input and checks if there are any collisions between any two rectangles. A rectangle is represented as a tuple (x, y, w, h) where (x, y) is the top-left corner of the rectangle, `w` is the width, and `h` is the height.\\n\\nThe function should return True if any pair of rectangles collide, and False otherwise. Use an iterative approach and check for collisions based on the bounding box collision detection algorithm. If a collision is found, return True immediately without checking for more collisions.']\n",rl,V,pl,E,cl,F,Be='Before sending the instructions to an LLM API, we need to format the instructions with the correct <code>chat_template</code> for each of the models we want to evaluate. This essentially entails wrapping some special tokens around the instructions. See the <a href="https://huggingface.co/docs/transformers/main/en/chat_templating" rel="nofollow">docs</a> on chat templates for details.',yl,S,dl,H,Ze=`First prompt formatted for mistralai/Mixtral-8x7B-Instruct-v0.1:

 <s>[INST] Write a Python function named \`get_value\` that takes a matrix (represented by a list of lists) and a tuple of indices, and returns the value at that index in the matrix. The function should handle index out of range errors by returning None. [/INST] 


First prompt formatted for meta-llama/Meta-Llama-3-70B-Instruct:

 &lt;|begin_of_text|&gt;&lt;|start_header_id|&gt;user&lt;|end_header_id|&gt;

Write a Python function named \`get_value\` that takes a matrix (represented by a list of lists) and a tuple of indices, and returns the value at that index in the matrix. The function should handle index out of range errors by returning None.&lt;|eot_id|&gt;&lt;|start_header_id|&gt;assistant&lt;|end_header_id|&gt;
</s>`,jl,_,ml,x,Ae='Now we can send the instructions to the APIs for both LLMs to get outputs we can evaluate. We first define some parameters for generating the responses correctly. Hugging Face’s LLM APIs are powered by <a href="https://huggingface.co/docs/text-generation-inference/index" rel="nofollow">Text Generation Inference (TGI)</a> containers. See the TGI OpenAPI specifications <a href="https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/generate" rel="nofollow">here</a> and the explanations of different parameters in the Transformers Generation Parameters <a href="https://huggingface.co/docs/transformers/v4.30.0/main_classes/text_generation#transformers.GenerationConfig" rel="nofollow">docs</a>.',Jl,z,hl,$,Re='Now we can make a standard API request to the Serverless Inference API (<a href="https://huggingface.co/docs/api-inference/index" rel="nofollow">docs</a>). Note that the Serverless Inference API is mostly for testing and is rate limited. For testing without rate limits, you can create your own API via the HF Dedicated Endpoints (<a href="https://huggingface.co/docs/inference-endpoints/index" rel="nofollow">docs</a>). See also our corresponding tutorials in the <a href="https://huggingface.co/learn/cookbook/index" rel="nofollow">Open Source AI Cookbook</a>.',ul,w,Ul,q,wl,L,ke=`---First generation of mistralai/Mixtral-8x7B-Instruct-v0.1:
Here's a Python function that meets your requirements:

\`\`\`python
def get_value(matrix, indices):
    try:
        return matrix[indices[0]][indices[1]]
    except IndexError:
        return None
\`\`\`

This function takes a matrix (represented by a list of lists) and a tuple of indices as input. It first tries to access the value at the given indices in the matrix. If the indices are out of range, it catches the \`IndexError\` exception and returns \`None\`.


---First generation of meta-llama/Meta-Llama-3-70B-Instruct:
Here is a Python function that does what you described:
\`\`\`
def get_value(matrix, indices):
    try:
        row, col = indices
        return matrix[row][col]
    except IndexError:
        return None
\`\`\`
Here's an explanation of how the function works:

1. The function takes two arguments: \`matrix\` (a list of lists) and \`indices\` (a tuple of two integers, representing the row and column indices).
2. The function tries to access the value at the specified indices using \`matrix[row][col]\`.
3. If the indices are out of range (i.e., \`row\` or \`col\` is greater than the length of the corresponding dimension of the matrix), an \`IndexError\` exception is raised.
4. The \`except\` block catches the \`IndexError\` exception and returns \`None\` instead of raising an error.

Here's an example usage of the function:
\`\`\`
matrix = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]

print(get_value(matrix, (0, 0)))  &num; prints 1
print(get_value(matrix, (1, 1)))  &num; prints 5
print(get_value(matrix, (3, 0)))  &num; prints None (out of range)
print(get_value(matrix, (0, 3)))  &num; prints None (out of range)
\`\`\`
I hope this helps! Let me know if you have any questions.
`,Tl,D,Il,P,We="We can now store the LLM outputs in a dataset together with the original instructions.",bl,K,fl,O,gl,tt,ve='We use <a href="https://argilla.io/" rel="nofollow">Argilla</a>, an open-source data annotation tool. We run Argilla via a HF Space, which you can set up with just a few clicks without any local setup. You can create the HF Argilla Space by following the instructions for Space creation <a href="https://huggingface.co/new-space?template=argilla%2Fargilla-template-space" rel="nofollow">here</a>. For more details on HF Argilla Spaces, see also the detailed <a href="https://huggingface.co/docs/hub/spaces-sdks-docker-argilla" rel="nofollow">docs</a>. If you want, you can also run Argilla locally via Argilla’s docker containers (see <a href="https://docs.argilla.io/en/latest/getting_started/quickstart_installation.html" rel="nofollow">Argilla docs</a>).',Cl,lt,Ne="<strong>Things to consider when creating the HF Argilla Space:</strong>",Gl,et,Xe="<li><p><strong>Persistent storage:</strong> While creating the Argilla Space, it is important to enable persistent storage for your Space, to make sure that all annotations are saved.</p></li> <li><p><strong>User management:</strong> While creating the Argilla Space, three users are created automatically: <code>owner</code>, <code>admin</code>, <code>annotator</code>. You can set their respective names, API keys and passwords for logging into the Argilla interface manually while creating the Space (see image below).</p></li>",Bl,T,Qe,Zl,st,Ye="If you leave these fields blank, the following default values will be used.",Al,at,Ve="<thead><tr><th><strong>Item</strong></th> <th><strong>Default Value</strong></th></tr></thead> <tbody><tr><td>OWNER_USERNAME</td> <td><code>owner</code></td></tr> <tr><td>ADMIN_USERNAME</td> <td><code>admin</code></td></tr> <tr><td>ANNOTATOR_USERNAME</td> <td><code>annotator</code></td></tr> <tr><td>PASSWORD (all)</td> <td><code>12345678</code></td></tr> <tr><td>OWNER_API_KEY</td> <td><code>owner.apikey</code></td></tr> <tr><td>ADMIN_API_KEY</td> <td><code>admin.apikey</code></td></tr> <tr><td>ARGILLA_WORKSPACE</td> <td><code>admin</code></td></tr></tbody>",Rl,nt,Ee='For more details on user management with Argilla see the <a href="https://docs.argilla.io/en/latest/getting_started/installation/configurations/user_management.html" rel="nofollow">docs here</a>.',kl,Mt,Fe="Once you’ve created the Argilla Space, you should see the following login screen in your browser and you can login with the password and username specified above.",Wl,ot,Se,vl,it,Nl,rt,He="Before we can tailor the interface to our specific task and upload data, we need to first set up a few things.",Xl,pt,_e="<strong>Connecting this notebook to Argilla:</strong> We can now connect this notebook to Argilla to programmatically configure the interface and upload/download data.",Ql,ct,Yl,yt,Vl,dt,El,jt,xe="Writing good guidelines for your human annotators is just as important (and difficult) as writing good training code. Good instructions should fulfill the following criteria:",Fl,mt,ze="<li><strong>Simple and clear</strong>: The guidelines should be simple and clear to understand for people who do not know anything about your task yet. Always ask at least one colleague to reread the guidelines to make sure that there are no ambiguities.</li> <li><strong>Reproducible and explicit</strong>: All information for doing the annotation task should be contained in the guidelines. A common mistake is to create informal interpretations of the guidelines during conversations with selected annotators. Future annotators will not have this information and might do the task differently than intended if it is not made explicit in the guidelines.</li> <li><strong>Short and comprehensive</strong>: The guidelines should as short as possible, while containing all necessary information. Annotators tend not to read long guidelines properly, so try to keep them as short as possible, while remaining comprehensive.</li>",Sl,Jt,$e='Note that creating annotator guidelines is an iterative process. It is good practice to do a few dozen annotations yourself and refine the guidelines based on your learnings from the data before assigning the task to others. Versioning the guidelines can also help as the task evolves over time. See further tips in this <a href="https://argilla.io/blog/annotation-guidelines-practices/" rel="nofollow">blog post</a>.',Hl,ht,_l,ut,qe="<strong>Cumulative ratings vs. Likert scales:</strong> Note that the guidelines above ask the annotators to do cumulative ratings by adding points for explicit criteria. An alternative approach are “Likert scales”, where annotators are asked to rate responses on a continuous scale e.g. from 1 (very bad) to 3 (mediocre) to 5 (very good). We generally recommend cumulative ratings, because they force you and the annotators to make quality criteria explicit, while just rating a response as “4” (good) is ambiguous and will be interpreted differently by different annotators.",xl,Ut,zl,wt,Le="We can now can create our own <code>code-llm</code> task with it’s own interface tailored to the specific task.",$l,Tt,ql,It,De="After running the code above, you will see the new custom <code>code-llm</code> task in Argilla (and any other tasks you might have created before, see image).",Ll,bt,Pe,Dl,ft,Pl,gt,Ke='You can also read the <a href="https://docs.argilla.io/en/latest/conceptual_guides/llm/llm.html" rel="nofollow">detailed guide</a> on working with LLM data in Argilla for more guidance on creating different interfaces for different tasks.',Kl,Ct,Ol,Gt,Oe="At this point, the task is still empty. Let’s upload some data into the task interface with the code below.",te,Bt,le,Zt,ts="<strong>The final annotation interface</strong> will look similar to this:",ee,At,ls,se,Rt,es='<strong>Assign tasks to annotators</strong>: Argilla supports assigning tasks to multiple users/annotators. There are different ways of implementing task assignments, <a href="https://docs.argilla.io/en/latest/practical_guides/assign_records.html" rel="nofollow">documented here</a>. For this tutorial, we use the simplest metadata method, where everyone has access to the same full dataset and all annotations (via the <code>annotators</code> variable created above). To access the annotations assigned to them, an annotator then needs to use the <code>Metadata</code> filter in the interface to filter the data to only see records assigned to them (see image below). For larger teams and to get multiple annotations for the same record, it is better to use other task assignment methods.',ae,I,ss,ne,kt,Me,Wt,as="That’s it, we’ve created our custom data annotation interface with Argilla and we can now start annotating!",oe,vt,ns="<strong>Important</strong>: If you use Argilla in a HF Space, you need to activate persistent storage so that your data is safely stored and not automatically deleted after a while. For production settings, make sure that persistent storage is activated <strong>before</strong> making any annotations to avoid data loss.",ie,Nt,re,Xt,Ms='After annotating, you can pull the data from Argilla and simply store and process them locally in any tabular format (see <a href="https://docs.argilla.io/en/latest/practical_guides/export_dataset.html" rel="nofollow">docs here</a>). You can also download filtered version of the dataset (<a href="https://docs.argilla.io/en/latest/tutorials_and_integrations/tutorials/feedback/end2end_examples/filter-and-query-008.html" rel="nofollow">docs</a>).',pe,Qt,ce,Yt,ye,Vt,de,Et,os="That’s it! You’ve created synthetic LLM data with the HF inference API, created your custom annotation interface with Argilla, uploaded the LLM data into Argilla, evaluated/corrected the data, and after annotation you have downloaded the data in a simple tabular format for downstream use.",je,Ft,is="We have specifically designed the pipeline and the interface for <strong>two main use-cases</strong>:",me,St,rs='<li>Evaluation: You can now simply use the numeric scores in the <code>score_response_1</code> and <code>score_response_2</code> columns to calculate which model was better overall. You can also inspect responses with very low or high ratings for a detailed error analysis. As you test or train different models, you can reuse this pipeline and track improvements of different models over time.</li> <li>Training: After annotating enough data, you can create a train-test split from the data and fine-tune your own model. You can either use highly rated response texts for supervised fine-tuning with the the <a href="https://huggingface.co/docs/trl/en/sft_trainer" rel="nofollow">TRL SFTTrainer</a>, or you can directly use the ratings for preference-tuning techniques like DPO with the <a href="https://huggingface.co/docs/trl/en/dpo_trainer" rel="nofollow">TRL DPOTrainer</a>. See the <a href="https://huggingface.co/docs/trl/en/index" rel="nofollow">TRL docs</a> for the pros and cons of different LLM fine-tuning techniques.</li>',Je,Ht,ps='<strong>Adapt and improve:</strong> Many things can be improved to tailor this pipeline to your specific use-cases. For example, you can prompt an LLM to evaluate the outputs of the two LLMs with instructions very similar to the guidelines for human annotators (“LLM-as-a-judge” approach). This can help further speed up your evaluation pipeline. See our <a href="https://huggingface.co/learn/cookbook/llm_judge" rel="nofollow">LLM-as-a-judge recipe</a> for an example implementation of LLM-as-a-judge and our overall <a href="https://huggingface.co/learn/cookbook/index" rel="nofollow">Open-Source AI Cookbook</a> for many other ideas.',he,_t,ue,zt,Ue;return f=new Ts({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/rag_with_sql_reranker/notebooks/en/enterprise_cookbook_argilla.ipynb"}]}}),g=new h({props:{title:"Data Annotation with Argilla Spaces",local:"data-annotation-with-argilla-spaces",headingTag:"h1"}}),k=new h({props:{title:"Install required packages and connect to HF Hub",local:"install-required-packages-and-connect-to-hf-hub",headingTag:"h2"}}),W=new J({props:{code:"IXBpcCUyMGluc3RhbGwlMjAlMjJhcmdpbGxhJTVCc2VydmVyJTVEfiUzRDEuMjcuMCUyMiUwQSFwaXAlMjBpbnN0YWxsJTIwdHJhbnNmb3JtZXJzfiUzRDQuNDAuMCUwQSFwaXAlMjBpbnN0YWxsJTIwZGF0YXNldHN+JTNEMi4xOS4wJTBBIXBpcCUyMGluc3RhbGwlMjBodWdnaW5nZmFjZV9odWJ+JTNEMC4yMy4y",highlighted:`!pip install <span class="hljs-string">&quot;argilla[server]~=1.27.0&quot;</span>
!pip install transformers~=<span class="hljs-number">4.40</span><span class="hljs-number">.0</span>
!pip install datasets~=<span class="hljs-number">2.19</span><span class="hljs-number">.0</span>
!pip install huggingface_hub~=<span class="hljs-number">0.23</span><span class="hljs-number">.2</span>`,wrap:!1}}),v=new J({props:{code:"JTIzJTIwTG9naW4lMjB0byUyMHRoZSUyMEhGJTIwSHViLiUyMFdlJTIwcmVjb21tZW5kJTIwdXNpbmclMjB0aGlzJTIwbG9naW4lMjBtZXRob2QlMjAlMEElMjMlMjB0byUyMGF2b2lkJTIwdGhlJTIwbmVlZCUyMGZvciUyMGV4cGxpY2l0bHklMjBzdG9yaW5nJTIweW91ciUyMEhGJTIwdG9rZW4lMjBpbiUyMHZhcmlhYmxlcyUyMCUwQWltcG9ydCUyMGh1Z2dpbmdmYWNlX2h1YiUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmUlMEFodWdnaW5nZmFjZV9odWIubG9naW4oYWRkX3RvX2dpdF9jcmVkZW50aWFsJTNEVHJ1ZSk=",highlighted:`<span class="hljs-comment"># Login to the HF Hub. We recommend using this login method </span>
<span class="hljs-comment"># to avoid the need for explicitly storing your HF token in variables </span>
<span class="hljs-keyword">import</span> huggingface_hub
!git config --<span class="hljs-keyword">global</span> credential.helper store
huggingface_hub.login(add_to_git_credential=<span class="hljs-literal">True</span>)`,wrap:!1}}),N=new h({props:{title:"Download example task data",local:"download-example-task-data",headingTag:"h2"}}),Q=new J({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBJTIzJTIwdGFraW5nJTIwYSUyMHNtYWxsJTIwc2FtcGxlJTIwaGVyZSUyMGZvciUyMGZhc3RlciUyMHRlc3RpbmclMEFkYXRhc2V0X2NvZGV0YXNrJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmJpZ2NvZGUlMkZzZWxmLW9zcy1pbnN0cnVjdC1zYzItZXhlYy1maWx0ZXItNTBrJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiU1QiUzQTMlNUQlMjIpJTBBcHJpbnQoJTIyRGF0YXNldCUyMHN0cnVjdHVyZSUzQSU1Q24lMjIlMkMlMjBkYXRhc2V0X2NvZGV0YXNrJTJDJTIwJTIyJTVDbiUyMiklMEElMEElMjMlMjBXZSUyMGFyZSUyMGFyZSUyMG9ubHklMjBpbnRlcmVzdGVkJTIwaW4lMjB0aGUlMjBpbnN0cnVjdGlvbnMlMkZwcm9tcHRzJTIwcHJvdmlkZWQlMjBpbiUyMHRoZSUyMGRhdGFzZXQlMEFpbnN0cnVjdGlvbnNfbHN0JTIwJTNEJTIwZGF0YXNldF9jb2RldGFzayU1QiUyMmluc3RydWN0aW9uJTIyJTVEJTBBcHJpbnQoJTIyRXhhbXBsZSUyMGluc3RydWN0aW9ucyUzQSU1Q24lMjIlMkMlMjBpbnN0cnVjdGlvbnNfbHN0JTVCJTNBMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># taking a small sample here for faster testing</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset_codetask = load_dataset(<span class="hljs-string">&quot;bigcode/self-oss-instruct-sc2-exec-filter-50k&quot;</span>, split=<span class="hljs-string">&quot;train[:3]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Dataset structure:\\n&quot;</span>, dataset_codetask, <span class="hljs-string">&quot;\\n&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We are are only interested in the instructions/prompts provided in the dataset</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>instructions_lst = dataset_codetask[<span class="hljs-string">&quot;instruction&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Example instructions:\\n&quot;</span>, instructions_lst[:<span class="hljs-number">2</span>])`,wrap:!1}}),V=new h({props:{title:"Prompt two LLMs on the example task",local:"prompt-two-llms-on-the-example-task",headingTag:"h2"}}),E=new h({props:{title:"Formatting the instructions with a chat_template",local:"formatting-the-instructions-with-a-chattemplate",headingTag:"h4"}}),S=new J({props:{code:"JTIzJTIwYXBwbHklMjBjb3JyZWN0JTIwY2hhdCUyMGZvcm1hdHRpbmclMjB0byUyMGluc3RydWN0aW9ucyUyMGZyb20lMjBkYXRhc2V0JTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbHNfdG9fY29tcGFyZSUyMCUzRCUyMCU1QiUyMm1pc3RyYWxhaSUyRk1peHRyYWwtOHg3Qi1JbnN0cnVjdC12MC4xJTIyJTJDJTIwJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy03MEItSW5zdHJ1Y3QlMjIlNUQlMEElMEElMEFkZWYlMjBmb3JtYXRfcHJvbXB0KHByb21wdCUyQyUyMHRva2VuaXplciklM0ElMEElMjAlMjAlMjAlMjBtZXNzYWdlcyUyMCUzRCUyMCU1QiU3QiUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMHByb21wdCU3RCU1RCUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzX3Rva2VuaXplZCUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1lc3NhZ2VzJTJDJTIwdG9rZW5pemUlM0RGYWxzZSUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMG1lc3NhZ2VzX3Rva2VuaXplZCUwQSUwQSUwQXByb21wdHNfZm9ybWF0dGVkX2RpYyUyMCUzRCUyMCU3QiU3RCUwQWZvciUyMG1vZGVsJTIwaW4lMjBtb2RlbHNfdG9fY29tcGFyZSUzQSUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsKSUwQSUwQSUyMCUyMCUyMCUyMHByb21wdF9mb3JtYXR0ZWQlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpbnN0cnVjdGlvbiUyMGluJTIwaW5zdHJ1Y3Rpb25zX2xzdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb21wdF9mb3JtYXR0ZWQuYXBwZW5kKGZvcm1hdF9wcm9tcHQoaW5zdHJ1Y3Rpb24lMkMlMjB0b2tlbml6ZXIpKSUwQSUwQSUyMCUyMCUyMCUyMHByb21wdHNfZm9ybWF0dGVkX2RpYy51cGRhdGUoJTdCbW9kZWwlM0ElMjBwcm9tcHRfZm9ybWF0dGVkJTdEKSUwQSUwQSUwQXByaW50KCUwQSUyMCUyMCUyMCUyMGYlMjIlNUNuRmlyc3QlMjBwcm9tcHQlMjBmb3JtYXR0ZWQlMjBmb3IlMjAlN0Jtb2RlbHNfdG9fY29tcGFyZSU1QjAlNUQlN0QlM0ElNUNuJTVDbiUyMiUyQyUwQSUyMCUyMCUyMCUyMHByb21wdHNfZm9ybWF0dGVkX2RpYyU1Qm1vZGVsc190b19jb21wYXJlJTVCMCU1RCU1RCU1QjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjIlNUNuJTVDbiUyMiUyQyUwQSklMEFwcmludCglMEElMjAlMjAlMjAlMjBmJTIyRmlyc3QlMjBwcm9tcHQlMjBmb3JtYXR0ZWQlMjBmb3IlMjAlN0Jtb2RlbHNfdG9fY29tcGFyZSU1QjElNUQlN0QlM0ElNUNuJTVDbiUyMiUyQyUwQSUyMCUyMCUyMCUyMHByb21wdHNfZm9ybWF0dGVkX2RpYyU1Qm1vZGVsc190b19jb21wYXJlJTVCMSU1RCU1RCU1QjAlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjIlNUNuJTVDbiUyMiUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># apply correct chat formatting to instructions from dataset</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>models_to_compare = [<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-Instruct-v0.1&quot;</span>, <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-70B-Instruct&quot;</span>]


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">format_prompt</span>(<span class="hljs-params">prompt, tokenizer</span>):
<span class="hljs-meta">... </span>    messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt}]
<span class="hljs-meta">... </span>    messages_tokenized = tokenizer.apply_chat_template(
<span class="hljs-meta">... </span>        messages, tokenize=<span class="hljs-literal">False</span>, add_generation_prompt=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> messages_tokenized


<span class="hljs-meta">&gt;&gt;&gt; </span>prompts_formatted_dic = {}
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model <span class="hljs-keyword">in</span> models_to_compare:
<span class="hljs-meta">... </span>    tokenizer = AutoTokenizer.from_pretrained(model)

<span class="hljs-meta">... </span>    prompt_formatted = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> instruction <span class="hljs-keyword">in</span> instructions_lst:
<span class="hljs-meta">... </span>        prompt_formatted.append(format_prompt(instruction, tokenizer))

<span class="hljs-meta">... </span>    prompts_formatted_dic.update({model: prompt_formatted})


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(
<span class="hljs-meta">... </span>    <span class="hljs-string">f&quot;\\nFirst prompt formatted for <span class="hljs-subst">{models_to_compare[<span class="hljs-number">0</span>]}</span>:\\n\\n&quot;</span>,
<span class="hljs-meta">... </span>    prompts_formatted_dic[models_to_compare[<span class="hljs-number">0</span>]][<span class="hljs-number">0</span>],
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;\\n\\n&quot;</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(
<span class="hljs-meta">... </span>    <span class="hljs-string">f&quot;First prompt formatted for <span class="hljs-subst">{models_to_compare[<span class="hljs-number">1</span>]}</span>:\\n\\n&quot;</span>,
<span class="hljs-meta">... </span>    prompts_formatted_dic[models_to_compare[<span class="hljs-number">1</span>]][<span class="hljs-number">0</span>],
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;\\n\\n&quot;</span>,
<span class="hljs-meta">... </span>)`,wrap:!1}}),_=new h({props:{title:"Sending the instructions to the HF Inference API",local:"sending-the-instructions-to-the-hf-inference-api",headingTag:"h4"}}),z=new J({props:{code:"Z2VuZXJhdGlvbl9wYXJhbXMlMjAlM0QlMjBkaWN0KCUwQSUyMCUyMCUyMCUyMCUyMyUyMHdlJTIwdXNlJTIwbG93JTIwdGVtcGVyYXR1ciUyMGFuZCUyMHRvcF9wJTIwdG8lMjByZWR1Y2UlMjBjcmVhdGl2aXR5JTIwYW5kJTIwaW5jcmVhc2UlMjBsaWtlbGlob29kJTIwb2YlMjBoaWdobHklMjBwcm9iYWJsZSUyMHRva2VucyUwQSUyMCUyMCUyMCUyMHRlbXBlcmF0dXJlJTNEMC4yJTJDJTBBJTIwJTIwJTIwJTIwdG9wX3AlM0QwLjYwJTJDJTBBJTIwJTIwJTIwJTIwdG9wX2slM0ROb25lJTJDJTBBJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNENTEyJTIwKiUyMDIlMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fZnVsbF90ZXh0JTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBzZWVkJTNENDIlMkMlMEElMjAlMjAlMjAlMjAlMjMlMjBkZXRhaWxzJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMyUyMHN0b3AlM0QlNUIlMjIlM0MlN0NFTkRfT0ZfVFVSTl9UT0tFTiU3QyUzRSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMyUyMGdyYW1tYXIlM0QlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyanNvbiUyMiU3RCUwQSUyMCUyMCUyMCUyMG1heF90aW1lJTNETm9uZSUyQyUwQSUyMCUyMCUyMCUyMHN0cmVhbSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwdXNlX2NhY2hlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjB3YWl0X2Zvcl9tb2RlbCUzREZhbHNlJTJDJTBBKQ==",highlighted:`generation_params = <span class="hljs-built_in">dict</span>(
    <span class="hljs-comment"># we use low temperatur and top_p to reduce creativity and increase likelihood of highly probable tokens</span>
    temperature=<span class="hljs-number">0.2</span>,
    top_p=<span class="hljs-number">0.60</span>,
    top_k=<span class="hljs-literal">None</span>,
    repetition_penalty=<span class="hljs-number">1.0</span>,
    do_sample=<span class="hljs-literal">True</span>,
    max_new_tokens=<span class="hljs-number">512</span> * <span class="hljs-number">2</span>,
    return_full_text=<span class="hljs-literal">False</span>,
    seed=<span class="hljs-number">42</span>,
    <span class="hljs-comment"># details=True,</span>
    <span class="hljs-comment"># stop=[&quot;&lt;|END_OF_TURN_TOKEN|&gt;&quot;],</span>
    <span class="hljs-comment"># grammar={&quot;type&quot;: &quot;json&quot;}</span>
    max_time=<span class="hljs-literal">None</span>,
    stream=<span class="hljs-literal">False</span>,
    use_cache=<span class="hljs-literal">False</span>,
    wait_for_model=<span class="hljs-literal">False</span>,
)`,wrap:!1}}),w=new ws({props:{warning:!1,$$slots:{default:[bs]},$$scope:{ctx:qt}}}),q=new J({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwdHFkbS5hdXRvJTIwaW1wb3J0JTIwdHFkbSUwQSUwQSUwQSUyMyUyMEhpbnQlM0ElMjB1c2UlMjBhc3luY2hyb25vdXMlMjBBUEklMjBjYWxscyUyMChhbmQlMjBkZWRpY2F0ZWQlMjBlbmRwb2ludHMpJTIwdG8lMjBpbmNyZWFzZSUyMHNwZWVkJTBBZGVmJTIwcXVlcnkocGF5bG9hZCUzRE5vbmUlMkMlMjBhcGlfdXJsJTNETm9uZSklM0ElMEElMjAlMjAlMjAlMjByZXNwb25zZSUyMCUzRCUyMHJlcXVlc3RzLnBvc3QoYXBpX3VybCUyQyUyMGhlYWRlcnMlM0RoZWFkZXJzJTJDJTIwanNvbiUzRHBheWxvYWQpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwcmVzcG9uc2UuanNvbigpJTBBJTBBJTBBaGVhZGVycyUyMCUzRCUyMCU3QiUyMkF1dGhvcml6YXRpb24lMjIlM0ElMjBmJTIyQmVhcmVyJTIwJTdCaHVnZ2luZ2ZhY2VfaHViLmdldF90b2tlbigpJTdEJTIyJTdEJTBBJTBBb3V0cHV0X2RpYyUyMCUzRCUyMCU3QiU3RCUwQWZvciUyMG1vZGVsJTIwaW4lMjBtb2RlbHNfdG9fY29tcGFyZSUzQSUwQSUyMCUyMCUyMCUyMCUyMyUyMGNyZWF0ZSUyMEFQSSUyMHVybHMlMjBmb3IlMjBlYWNoJTIwbW9kZWwlMEElMjAlMjAlMjAlMjAlMjMlMjBXaGVuJTIwdXNpbmclMjBkZWRpY2F0ZWQlMjBlbmRwb2ludHMlMkMlMjB5b3UlMjBjYW4lMjByZXVzZSUyMHRoZSUyMHNhbWUlMjBjb2RlJTIwYW5kJTIwc2ltcGx5JTIwcmVwbGFjZSUyMHRoaXMlMjBVUkwlMEElMjAlMjAlMjAlMjBhcGlfdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZhcGktaW5mZXJlbmNlLmh1Z2dpbmdmYWNlLmNvJTJGbW9kZWxzJTJGJTIyJTIwJTJCJTIwbW9kZWwlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBzZW5kJTIwcmVxdWVzdHMlMjB0byUyMEFQSSUwQSUyMCUyMCUyMCUyMG91dHB1dF9sc3QlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBwcm9tcHQlMjBpbiUyMHRxZG0ocHJvbXB0X2Zvcm1hdHRlZCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXQlMjAlM0QlMjBxdWVyeShwYXlsb2FkJTNEJTdCJTIyaW5wdXRzJTIyJTNBJTIwcHJvbXB0JTJDJTIwJTIycGFyYW1ldGVycyUyMiUzQSUyMCU3QioqZ2VuZXJhdGlvbl9wYXJhbXMlN0QlN0QlMkMlMjBhcGlfdXJsJTNEYXBpX3VybCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfbHN0LmFwcGVuZChvdXRwdXQlNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQpJTBBJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpYy51cGRhdGUoJTdCbW9kZWwlM0ElMjBvdXRwdXRfbHN0JTdEKSUwQSUwQSUyMyUyMGluc3BlY3QlMjBmaXJzdCUyMGdlbmVyYXRpb24lMEFwcmludChmJTIyLS0tRmlyc3QlMjBnZW5lcmF0aW9uJTIwb2YlMjAlN0Jtb2RlbHNfdG9fY29tcGFyZSU1QjAlNUQlN0QlM0ElNUNuJTdCb3V0cHV0X2RpYyU1Qm1vZGVsc190b19jb21wYXJlJTVCMCU1RCU1RCU1QjAlNUQlN0QlNUNuJTVDbiUyMiklMEFwcmludChmJTIyLS0tRmlyc3QlMjBnZW5lcmF0aW9uJTIwb2YlMjAlN0Jtb2RlbHNfdG9fY29tcGFyZSU1QjElNUQlN0QlM0ElNUNuJTdCb3V0cHV0X2RpYyU1Qm1vZGVsc190b19jb21wYXJlJTVCMSU1RCU1RCU1QjAlNUQlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Hint: use asynchronous API calls (and dedicated endpoints) to increase speed</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">query</span>(<span class="hljs-params">payload=<span class="hljs-literal">None</span>, api_url=<span class="hljs-literal">None</span></span>):
<span class="hljs-meta">... </span>    response = requests.post(api_url, headers=headers, json=payload)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> response.json()


<span class="hljs-meta">&gt;&gt;&gt; </span>headers = {<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;Bearer <span class="hljs-subst">{huggingface_hub.get_token()}</span>&quot;</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>output_dic = {}
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model <span class="hljs-keyword">in</span> models_to_compare:
<span class="hljs-meta">... </span>    <span class="hljs-comment"># create API urls for each model</span>
<span class="hljs-meta">... </span>    <span class="hljs-comment"># When using dedicated endpoints, you can reuse the same code and simply replace this URL</span>
<span class="hljs-meta">... </span>    api_url = <span class="hljs-string">&quot;https://api-inference.huggingface.co/models/&quot;</span> + model

<span class="hljs-meta">... </span>    <span class="hljs-comment"># send requests to API</span>
<span class="hljs-meta">... </span>    output_lst = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> tqdm(prompt_formatted):
<span class="hljs-meta">... </span>        output = query(payload={<span class="hljs-string">&quot;inputs&quot;</span>: prompt, <span class="hljs-string">&quot;parameters&quot;</span>: {**generation_params}}, api_url=api_url)
<span class="hljs-meta">... </span>        output_lst.append(output[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])

<span class="hljs-meta">... </span>    output_dic.update({model: output_lst})

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># inspect first generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;---First generation of <span class="hljs-subst">{models_to_compare[<span class="hljs-number">0</span>]}</span>:\\n<span class="hljs-subst">{output_dic[models_to_compare[<span class="hljs-number">0</span>]][<span class="hljs-number">0</span>]}</span>\\n\\n&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;---First generation of <span class="hljs-subst">{models_to_compare[<span class="hljs-number">1</span>]}</span>:\\n<span class="hljs-subst">{output_dic[models_to_compare[<span class="hljs-number">1</span>]][<span class="hljs-number">0</span>]}</span>&quot;</span>)`,wrap:!1}}),D=new h({props:{title:"Store the LLM outputs in a dataset",local:"store-the-llm-outputs-in-a-dataset",headingTag:"h4"}}),K=new J({props:{code:"JTIzJTIwY3JlYXRlJTIwYSUyMEhGJTIwZGF0YXNldCUyMHdpdGglMjB0aGUlMjBpbnN0cnVjdGlvbnMlMjBhbmQlMjBtb2RlbCUyMG91dHB1dHMlMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBEYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmluc3RydWN0aW9ucyUyMiUzQSUyMGluc3RydWN0aW9uc19sc3QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZXNwb25zZV9tb2RlbF8xJTIyJTNBJTIwb3V0cHV0X2RpYyU1Qm1vZGVsc190b19jb21wYXJlJTVCMCU1RCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlc3BvbnNlX21vZGVsXzIlMjIlM0ElMjBvdXRwdXRfZGljJTVCbW9kZWxzX3RvX2NvbXBhcmUlNUIxJTVEJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTBBKSUwQSUwQWRhdGFzZXQ=",highlighted:`<span class="hljs-comment"># create a HF dataset with the instructions and model outputs</span>
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

dataset = Dataset.from_dict(
    {
        <span class="hljs-string">&quot;instructions&quot;</span>: instructions_lst,
        <span class="hljs-string">&quot;response_model_1&quot;</span>: output_dic[models_to_compare[<span class="hljs-number">0</span>]],
        <span class="hljs-string">&quot;response_model_2&quot;</span>: output_dic[models_to_compare[<span class="hljs-number">1</span>]],
    }
)

dataset`,wrap:!1}}),O=new h({props:{title:"Create and configure your Argilla annotation interface",local:"create-and-configure-your-argilla-annotation-interface",headingTag:"h2"}}),it=new h({props:{title:"Programmatically interact with Argilla",local:"programmatically-interact-with-argilla",headingTag:"h4"}}),ct=new J({props:{code:"JTIzJTIwQWZ0ZXIlMjBzdGFydGluZyUyMHRoZSUyMEFyZ2lsbGElMjBTcGFjZSUyMChvciUyMGxvY2FsJTIwZG9ja2VyJTIwY29udGFpbmVyKSUyMHlvdSUyMGNhbiUyMGNvbm5lY3QlMjB0byUyMHRoZSUyMFNwYWNlJTIwd2l0aCUyMHRoZSUyMGNvZGUlMjBiZWxvdy4lMEElMjMlMjBIZXJlJTIwd2UlMjB1c2UlMjBBcmdpbGxhJTIwYXMlMjB0aGUlMjAlMjJvd25lciUyMiUyMHVzZXIlMEFpbXBvcnQlMjBhcmdpbGxhJTIwYXMlMjByZyUwQSUwQXJnLmluaXQoJTBBJTIwJTIwJTIwJTIwJTIzJTIwVGhlJTIwJTYwYXBpX3VybCU2MCUyMHRvJTIwdGhlJTIwc3BhY2UlMjBmb2xsb3dzJTIwdGhlJTIwcGF0dGVybiUyMCUyMmh0dHBzJTNBJTJGJTJGdXNlcm5hbWUtc3BhY2VuYW1lLmhmLnNwYWNlJTIyJTBBJTIwJTIwJTIwJTIwYXBpX3VybCUzRCUyMmh0dHBzJTNBJTJGJTJGbW9yaXR6bGF1cmVyLWFyZ2lsbGEtMDAuaGYuc3BhY2UlMjIlMkMlMjAlMjAlMjMlMjBGZiUyMHlvdSUyMHJ1biUyMEFyZ2lsbGElMjBsb2NhbGx5JTNBJTIwJTIyaHR0cCUzQSUyRiUyRmxvY2FsaG9zdCUzQTY5MDAlMjIlMEElMjAlMjAlMjAlMjBhcGlfa2V5JTNEJTIyb3duZXIuYXBpa2V5JTIyJTJDJTIwJTIwJTIzJTIwJTIyb3duZXIuYXBpa2V5JTIyJTJDJTIwJTIyYWRtaW4uYXBpa2V5JTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwVG8lMjB1c2UlMjBhJTIwcHJpdmF0ZSUyMEhGJTIwQXJnaWxsYSUyMFNwYWNlJTJDJTIwYWxzbyUyMHBhc3MlMjB5b3VyJTIwSEYlMjB0b2tlbiUwQSUyMCUyMCUyMCUyMGV4dHJhX2hlYWRlcnMlM0QlN0IlMjJBdXRob3JpemF0aW9uJTIyJTNBJTIwZiUyMkJlYXJlciUyMCU3Qmh1Z2dpbmdmYWNlX2h1Yi5nZXRfdG9rZW4oKSU3RCUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMHdvcmtzcGFjZSUzRCUyMmFkbWluJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-comment"># After starting the Argilla Space (or local docker container) you can connect to the Space with the code below.</span>
<span class="hljs-comment"># Here we use Argilla as the &quot;owner&quot; user</span>
<span class="hljs-keyword">import</span> argilla <span class="hljs-keyword">as</span> rg

rg.init(
    <span class="hljs-comment"># The \`api_url\` to the space follows the pattern &quot;https://username-spacename.hf.space&quot;</span>
    api_url=<span class="hljs-string">&quot;https://moritzlaurer-argilla-00.hf.space&quot;</span>,  <span class="hljs-comment"># Ff you run Argilla locally: &quot;http://localhost:6900&quot;</span>
    api_key=<span class="hljs-string">&quot;owner.apikey&quot;</span>,  <span class="hljs-comment"># &quot;owner.apikey&quot;, &quot;admin.apikey&quot;</span>
    <span class="hljs-comment"># To use a private HF Argilla Space, also pass your HF token</span>
    extra_headers={<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;Bearer <span class="hljs-subst">{huggingface_hub.get_token()}</span>&quot;</span>},
    workspace=<span class="hljs-string">&quot;admin&quot;</span>,
)`,wrap:!1}}),yt=new J({props:{code:"dXNlciUyMCUzRCUyMHJnLlVzZXIubWUoKSUwQXVzZXI=",highlighted:`user = rg.User.me()
user`,wrap:!1}}),dt=new h({props:{title:"Write good annotator guidelines",local:"write-good-annotator-guidelines",headingTag:"h4"}}),ht=new J({props:{code:"YW5ub3RhdG9yX2d1aWRlbGluZXMlMjAlM0QlMjAlMjIlMjIlMjIlNUMlMEFZb3VyJTIwdGFzayUyMGlzJTIwdG8lMjBldmFsdWF0ZSUyMHRoZSUyMHJlc3BvbnNlcyUyMG9mJTIwdHdvJTIwTExNcyUyMHRvJTIwY29kZSUyMGdlbmVyYXRpb24lMjB0YXNrcy4lMjAlMEElMEFGaXJzdCUyQyUyMHlvdSUyMG5lZWQlMjB0byUyMHNjb3JlJTIwZWFjaCUyMHJlc3BvbnNlJTIwb24lMjBhJTIwc2NhbGUlMjBmcm9tJTIwMCUyMHRvJTIwNy4lMjBZb3UlMjBhZGQlMjBwb2ludHMlMjB0byUyMHlvdXIlMjBmaW5hbCUyMHNjb3JlJTIwYmFzZWQlMjBvbiUyMHRoZSUyMGZvbGxvd2luZyUyMGNyaXRlcmlhJTNBJTBBLSUyMEFkZCUyMHVwJTIwdG8lMjAlMkIyJTIwcG9pbnRzJTJDJTIwaWYlMjB0aGUlMjBjb2RlJTIwaXMlMjBwcm9wZXJseSUyMGNvbW1lbnRlZCUyQyUyMHdpdGglMjBpbmxpbmUlMjBjb21tZW50cyUyMGFuZCUyMGRvYyUyMHN0cmluZ3MlMjBmb3IlMjBmdW5jdGlvbnMuJTBBLSUyMEFkZCUyMHVwJTIwdG8lMjAlMkIyJTIwcG9pbnRzJTJDJTIwaWYlMjB0aGUlMjBjb2RlJTIwY29udGFpbnMlMjBhJTIwZ29vZCUyMGV4YW1wbGUlMjBmb3IlMjB0ZXN0aW5nLiUyMCUwQS0lMjBBZGQlMjB1cCUyMHRvJTIwJTJCMyUyMHBvaW50cyUyQyUyMGlmJTIwdGhlJTIwY29kZSUyMHJ1bnMlMjBhbmQlMjB3b3JrcyUyMGNvcnJlY3RseS4lMjBDb3B5JTIwdGhlJTIwY29kZSUyMGludG8lMjBhbiUyMElERSUyMGFuZCUyMHRlc3QlMjBpdCUyMHdpdGglMjBhdCUyMGxlYXN0JTIwdHdvJTIwZGlmZmVyZW50JTIwaW5wdXRzLiUyMEF0dHJpYnV0ZSUyMG9uZSUyMHBvaW50JTIwaWYlMjB0aGUlMjBjb2RlJTIwaXMlMjBvdmVyYWxsJTIwY29ycmVjdCUyQyUyMGJ1dCUyMGhhcyUyMHNvbWUlMjBpc3N1ZXMuJTIwQXR0cmlidXRlJTIwdGhyZWUlMjBwb2ludHMlMjBpZiUyMHRoZSUyMGNvZGUlMjBpcyUyMGZ1bGx5JTIwY29ycmVjdCUyMGFuZCUyMHJvYnVzdCUyMGFnYWluc3QlMjBkaWZmZXJlbnQlMjBzY2VuYXJpb3MuJTIwJTBBWW91ciUyMHJlc3VsdGluZyUyMGZpbmFsJTIwc2NvcmUlMjBjYW4lMjBiZSUyMGFueSUyMHZhbHVlJTIwYmV0d2VlbiUyMDAlMjB0byUyMDcuJTIwJTBBJTBBSWYlMjBib3RoJTIwcmVzcG9uc2VzJTIwaGF2ZSUyMGElMjBmaW5hbCUyMHNjb3JlJTIwb2YlMjAlM0MlM0QlMjA0JTJDJTIwc2VsZWN0JTIwb25lJTIwcmVzcG9uc2UlMjBhbmQlMjBjb3JyZWN0JTIwaXQlMjBtYW51YWxseSUyMGluJTIwdGhlJTIwdGV4dCUyMGZpZWxkLiUyMCUwQVRoZSUyMGNvcnJlY3RlZCUyMHJlc3BvbnNlJTIwbXVzdCUyMGZ1bGZpbGwlMjBhbGwlMjBjcml0ZXJpYSUyMGZyb20lMjBhYm92ZS4lMjAlMEElMjIlMjIlMjIlMEElMEFyYXRpbmdfdG9vbHRpcCUyMCUzRCUyMCUyMiUyMiUyMiU1QyUwQS0lMjBBZGQlMjB1cCUyMHRvJTIwJTJCMiUyMHBvaW50cyUyQyUyMGlmJTIwdGhlJTIwY29kZSUyMGlzJTIwcHJvcGVybHklMjBjb21tZW50ZWQlMkMlMjB3aXRoJTIwaW5saW5lJTIwY29tbWVudHMlMjBhbmQlMjBkb2MlMjBzdHJpbmdzJTIwZm9yJTIwZnVuY3Rpb25zLiUwQS0lMjBBZGQlMjB1cCUyMHRvJTIwJTJCMiUyMHBvaW50cyUyQyUyMGlmJTIwdGhlJTIwY29kZSUyMGNvbnRhaW5zJTIwYSUyMGdvb2QlMjBleGFtcGxlJTIwZm9yJTIwdGVzdGluZy4lMjAlMEEtJTIwQWRkJTIwdXAlMjB0byUyMCUyQjMlMjBwb2ludHMlMkMlMjBpZiUyMHRoZSUyMGNvZGUlMjBydW5zJTIwYW5kJTIwd29ya3MlMjBjb3JyZWN0bHkuJTIwQ29weSUyMHRoZSUyMGNvZGUlMjBpbnRvJTIwYW4lMjBJREUlMjBhbmQlMjB0ZXN0JTIwaXQlMjB3aXRoJTIwYXQlMjBsZWFzdCUyMHR3byUyMGRpZmZlcmVudCUyMGlucHV0cy4lMjBBdHRyaWJ1dGUlMjBvbmUlMjBwb2ludCUyMGlmJTIwdGhlJTIwY29kZSUyMHdvcmtzJTIwbW9zdGx5JTIwY29ycmVjdGx5JTJDJTIwYnV0JTIwaGFzJTIwc29tZSUyMGlzc3Vlcy4lMjBBdHRyaWJ1dGUlMjB0aHJlZSUyMHBvaW50cyUyMGlmJTIwdGhlJTIwY29kZSUyMGlzJTIwZnVsbHklMjBjb3JyZWN0JTIwYW5kJTIwcm9idXN0JTIwYWdhaW5zdCUyMGRpZmZlcmVudCUyMHNjZW5hcmlvcy4lMjAlMEElMjIlMjIlMjI=",highlighted:`annotator_guidelines = <span class="hljs-string">&quot;&quot;&quot;\\
Your task is to evaluate the responses of two LLMs to code generation tasks. 

First, you need to score each response on a scale from 0 to 7. You add points to your final score based on the following criteria:
- Add up to +2 points, if the code is properly commented, with inline comments and doc strings for functions.
- Add up to +2 points, if the code contains a good example for testing. 
- Add up to +3 points, if the code runs and works correctly. Copy the code into an IDE and test it with at least two different inputs. Attribute one point if the code is overall correct, but has some issues. Attribute three points if the code is fully correct and robust against different scenarios. 
Your resulting final score can be any value between 0 to 7. 

If both responses have a final score of &lt;= 4, select one response and correct it manually in the text field. 
The corrected response must fulfill all criteria from above. 
&quot;&quot;&quot;</span>

rating_tooltip = <span class="hljs-string">&quot;&quot;&quot;\\
- Add up to +2 points, if the code is properly commented, with inline comments and doc strings for functions.
- Add up to +2 points, if the code contains a good example for testing. 
- Add up to +3 points, if the code runs and works correctly. Copy the code into an IDE and test it with at least two different inputs. Attribute one point if the code works mostly correctly, but has some issues. Attribute three points if the code is fully correct and robust against different scenarios. 
&quot;&quot;&quot;</span>`,wrap:!1}}),Ut=new h({props:{title:"Tailor the Argilla interface to your specific task",local:"tailor-the-argilla-interface-to-your-specific-task",headingTag:"h4"}}),Tt=new J({props:{code:" ",highlighted:`dataset_argilla_name = <span class="hljs-string">&quot;code-llm&quot;</span>
reuse_existing_dataset = <span class="hljs-literal">False</span>  <span class="hljs-comment"># for easier iterative testing</span>

<span class="hljs-comment"># Create annotator groups. Used for task asignment via meta data filtering.</span>
<span class="hljs-comment"># See explanations on annotator task assignment further below</span>
annotators = [<span class="hljs-string">&quot;annotator-1&quot;</span>, <span class="hljs-string">&quot;annotator-2&quot;</span>, <span class="hljs-string">&quot;annotator-3&quot;</span>]

<span class="hljs-comment"># Create the interface structure via an Argilla FeedbackDataset</span>
dataset_argilla = rg.FeedbackDataset(
    <span class="hljs-comment"># The overall annotation guidelines, which human annotators can refer back to inside of the interface</span>
    guidelines=annotator_guidelines,
    <span class="hljs-comment"># The fields on the left side of the interface</span>
    fields=[
        rg.TextField(name=<span class="hljs-string">&quot;instruction&quot;</span>, title=<span class="hljs-string">&quot;Instruction:&quot;</span>, use_markdown=<span class="hljs-literal">True</span>, required=<span class="hljs-literal">True</span>),
        rg.TextField(name=<span class="hljs-string">&quot;generation_1&quot;</span>, title=<span class="hljs-string">&quot;Response model 1:&quot;</span>, use_markdown=<span class="hljs-literal">True</span>, required=<span class="hljs-literal">True</span>),
        rg.TextField(name=<span class="hljs-string">&quot;generation_2&quot;</span>, title=<span class="hljs-string">&quot;Response model 2:&quot;</span>, use_markdown=<span class="hljs-literal">True</span>, required=<span class="hljs-literal">True</span>),
    ],
    <span class="hljs-comment"># The different questions on the right side of the interface</span>
    <span class="hljs-comment"># These are the questions we ask annotators about the fields on the left of the interface</span>
    <span class="hljs-comment"># The available question types are documented here: https://docs.argilla.io/en/latest/getting_started/cheatsheet.html#configure-datasets</span>
    questions=[
        rg.RatingQuestion(
            name=<span class="hljs-string">&quot;score_response_1&quot;</span>,
            title=<span class="hljs-string">&quot;Your score for the response of model 1:&quot;</span>,
            description=rating_tooltip,  <span class="hljs-comment"># &quot;1 = very bad\\n2 = bad\\n3 = mediocre\\n4 = good\\n5 = very good&quot;,</span>
            <span class="hljs-comment"># Note: Argilla version &lt;= 1.28 does not yet support rating values of 0.</span>
            <span class="hljs-comment"># This will be possible starting version &gt;= 1.29</span>
            values=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>],
            required=<span class="hljs-literal">True</span>,
        ),
        rg.RatingQuestion(
            name=<span class="hljs-string">&quot;score_response_2&quot;</span>,
            title=<span class="hljs-string">&quot;Your score for the response of model 2:&quot;</span>,
            description=rating_tooltip,  <span class="hljs-comment"># &quot;1 = very bad\\n2 = bad\\n3 = mediocre\\n4 = good\\n5 = very good&quot;,</span>
            values=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>],
            required=<span class="hljs-literal">True</span>,
        ),
        rg.LabelQuestion(
            name=<span class="hljs-string">&quot;which_response_corrected&quot;</span>,
            title=<span class="hljs-string">&quot;If both responses score below 4, select a response to correct:&quot;</span>,
            description=<span class="hljs-string">&quot;Select the response you will correct in the text field below.&quot;</span>,
            labels=[<span class="hljs-string">&quot;Response 1&quot;</span>, <span class="hljs-string">&quot;Response 2&quot;</span>, <span class="hljs-string">&quot;Combination of both&quot;</span>, <span class="hljs-string">&quot;Neither&quot;</span>],
            required=<span class="hljs-literal">False</span>,
        ),
        rg.TextQuestion(
            name=<span class="hljs-string">&quot;correction&quot;</span>,
            title=<span class="hljs-string">&quot;Paste the selected response below and correct it manually:&quot;</span>,
            description=<span class="hljs-string">&quot;Your corrected response must fulfill all criteria from the annotation guidelines.&quot;</span>,
            use_markdown=<span class="hljs-literal">True</span>,
            required=<span class="hljs-literal">False</span>,
        ),
        rg.TextQuestion(
            name=<span class="hljs-string">&quot;comments&quot;</span>,
            title=<span class="hljs-string">&quot;Annotator Comments&quot;</span>,
            description=<span class="hljs-string">&quot;Add any additional comments here. E.g.: edge cases, issues with the interface etc.&quot;</span>,
            use_markdown=<span class="hljs-literal">True</span>,
            required=<span class="hljs-literal">False</span>,
        ),
    ],
    metadata_properties=[
        rg.TermsMetadataProperty(
            name=<span class="hljs-string">&quot;annotator-groups&quot;</span>,
            title=<span class="hljs-string">&quot;Annotator groups&quot;</span>,
            values=annotators,
        ),
        rg.TermsMetadataProperty(
            name=<span class="hljs-string">&quot;source-dataset&quot;</span>,
            title=<span class="hljs-string">&quot;Original dataset source&quot;</span>,
        ),
    ],
    allow_extra_metadata=<span class="hljs-literal">False</span>,
)


<span class="hljs-keyword">if</span> reuse_existing_dataset:
    dataset_argilla = rg.FeedbackDataset.from_argilla(dataset_argilla_name, workspace=<span class="hljs-string">&quot;admin&quot;</span>)
<span class="hljs-keyword">else</span>:
    <span class="hljs-comment"># check if dataset already exists</span>
    dataset_existing = [dataset <span class="hljs-keyword">for</span> dataset <span class="hljs-keyword">in</span> rg.list_datasets() <span class="hljs-keyword">if</span> dataset.name == dataset_argilla_name]
    <span class="hljs-comment"># if it already exists, delete it</span>
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(dataset_existing) &gt; <span class="hljs-number">0</span>:
        rg.FeedbackDataset.from_argilla(name=dataset_argilla_name, workspace=<span class="hljs-string">&quot;admin&quot;</span>).delete()
    <span class="hljs-comment"># push (updated) dataset to argilla</span>
    dataset_argilla = dataset_argilla.push_to_argilla(dataset_argilla_name, workspace=<span class="hljs-string">&quot;admin&quot;</span>)`,wrap:!1}}),ft=new J({props:{code:"JTIzJTIwVGhlJTIwZmluYWwlMjBhcmdpbGxhJTIwZGF0YXNldCUwQSUyMyUyMHByaW50KGRhdGFzZXRfYXJnaWxsYSk=",highlighted:`<span class="hljs-comment"># The final argilla dataset</span>
<span class="hljs-comment"># print(dataset_argilla)</span>`,wrap:!1}}),Ct=new h({props:{title:"Upload data to Argilla for our task",local:"upload-data-to-argilla-for-our-task",headingTag:"h4"}}),Bt=new J({props:{code:"aW1wb3J0JTIwcmFuZG9tJTBBJTBBJTIzJTIwSXRlcmF0ZSUyMG92ZXIlMjB0aGUlMjBzYW1wbGVzJTIwaW4lMjB0aGUlMjBkYXRhc2V0JTBBcmVjb3JkcyUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMGV4YW1wbGUlMjBpbiUyMGRhdGFzZXQlM0ElMEElMEElMjAlMjAlMjAlMjAlMjMlMjBBZGQlMjB0aGUlMjByZWNvcmRzJTIwdG8lMjB0aGUlMjBGZWVkYmFja0RhdGFzZXQlMEElMjAlMjAlMjAlMjByZWNvcmQlMjAlM0QlMjByZy5GZWVkYmFja1JlY29yZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmaWVsZHMlM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJpbnN0cnVjdGlvbiUyMiUzQSUyMGV4YW1wbGUlNUIlMjJpbnN0cnVjdGlvbnMlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJnZW5lcmF0aW9uXzElMjIlM0ElMjBleGFtcGxlJTVCJTIycmVzcG9uc2VfbW9kZWxfMSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmdlbmVyYXRpb25fMiUyMiUzQSUyMGV4YW1wbGUlNUIlMjJyZXNwb25zZV9tb2RlbF8yJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWV0YWRhdGElM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjB3ZSUyMHJhbmRvbWx5JTIwYXNzaWduJTIwYSUyMHJlY29yZCUyRnRhc2slMjB0byUyMHRoZSUyMGFubm90YXRvcnMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJhbm5vdGF0b3ItZ3JvdXBzJTIyJTNBJTIwcmFuZG9tLmNob2ljZShhbm5vdGF0b3JzKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnNvdXJjZS1kYXRhc2V0JTIyJTNBJTIwJTIyYmlnY29kZSUyRnNlbGYtb3NzLWluc3RydWN0LXNjMi1leGVjLWZpbHRlci01MGslMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwT3B0aW9uYWwlM0ElMjBhZGQlMjBwcmVmaWxsZWQlMjBzdWdnZXN0aW9uJTBBJTIwJTIwJTIwJTIwJTIzJTIweW91JTIwY2FuJTIwdXNlJTIwdGhpcyUyMHRvJTIwZmlsbCUyMFF1ZXN0aW9ucyUyMHdpdGglMjBzdWdnZXN0aW9ucyUyMGZyb20lMjBhbiUyMExMTS1hcy1hLWp1ZGdlJTIwc3lzdGVtJTBBJTIwJTIwJTIwJTIwJTIzJTIwdG8lMjBmdXJ0aGVyJTIwc3BlZWQlMjB1cCUyMG1hbnVhbCUyMGFubm90YXRpb24lMEElMjAlMjAlMjAlMjAlMjMlMjByZWNvcmQuc3VnZ2VzdGlvbnMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjMlMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjMlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbl9uYW1lJTIyJTNBJTIwJTIyc2NvcmVfcmVzcG9uc2VfMSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnZhbHVlJTIyJTNBJTIwZXhhbXBsZSU1QiUyMmxsbV9qdWRnZV9yYXRpbmclMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjMlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJhZ2VudCUyMiUzQSUyMCUyMmxsYW1hLTMtNzBiLWluc3RydWN0JTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIzJTIwJTVEJTBBJTBBJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGF0YXNldF9hcmdpbGxhLmFkZF9yZWNvcmRzKHJlY29yZCUyQyUyMHNob3dfcHJvZ3Jlc3MlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMGV4Y2VwdCUyMEV4Y2VwdGlvbiUyMGFzJTIwZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KCUyMkV4Y2VwdGlvbiUzQSUyMiUyQyUyMGUp",highlighted:`<span class="hljs-keyword">import</span> random

<span class="hljs-comment"># Iterate over the samples in the dataset</span>
records = []
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> dataset:

    <span class="hljs-comment"># Add the records to the FeedbackDataset</span>
    record = rg.FeedbackRecord(
        fields={
            <span class="hljs-string">&quot;instruction&quot;</span>: example[<span class="hljs-string">&quot;instructions&quot;</span>],
            <span class="hljs-string">&quot;generation_1&quot;</span>: example[<span class="hljs-string">&quot;response_model_1&quot;</span>],
            <span class="hljs-string">&quot;generation_2&quot;</span>: example[<span class="hljs-string">&quot;response_model_2&quot;</span>],
        },
        metadata={
            <span class="hljs-comment"># we randomly assign a record/task to the annotators</span>
            <span class="hljs-string">&quot;annotator-groups&quot;</span>: random.choice(annotators),
            <span class="hljs-string">&quot;source-dataset&quot;</span>: <span class="hljs-string">&quot;bigcode/self-oss-instruct-sc2-exec-filter-50k&quot;</span>,
        },
    )

    <span class="hljs-comment"># Optional: add prefilled suggestion</span>
    <span class="hljs-comment"># you can use this to fill Questions with suggestions from an LLM-as-a-judge system</span>
    <span class="hljs-comment"># to further speed up manual annotation</span>
    <span class="hljs-comment"># record.suggestions = [</span>
    <span class="hljs-comment">#    {</span>
    <span class="hljs-comment">#        &quot;question_name&quot;: &quot;score_response_1&quot;,</span>
    <span class="hljs-comment">#        &quot;value&quot;: example[&quot;llm_judge_rating&quot;],</span>
    <span class="hljs-comment">#        &quot;agent&quot;: &quot;llama-3-70b-instruct&quot;</span>
    <span class="hljs-comment">#    },</span>
    <span class="hljs-comment"># ]</span>

    <span class="hljs-keyword">try</span>:
        dataset_argilla.add_records(record, show_progress=<span class="hljs-literal">True</span>)
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Exception:&quot;</span>, e)`,wrap:!1}}),kt=new h({props:{title:"Annotate",local:"annotate",headingTag:"h2"}}),Nt=new h({props:{title:"Download annotated data",local:"download-annotated-data",headingTag:"h2"}}),Qt=new J({props:{code:"cmVtb3RlX2RhdGFzZXQlMjAlM0QlMjByZy5GZWVkYmFja0RhdGFzZXQuZnJvbV9hcmdpbGxhKGRhdGFzZXRfYXJnaWxsYV9uYW1lJTJDJTIwd29ya3NwYWNlJTNEJTIyYWRtaW4lMjIpJTBBJTBBJTIzJTIwcHVsbCUyMHRoZSUyMGZpcnN0JTIwTiUyMHJlY29yZHMlMjBmcm9tJTIwdGhlJTIwcmVtb3RlJTIwZGF0YXNldCUwQWxvY2FsX2RhdGFzZXQlMjAlM0QlMjByZW1vdGVfZGF0YXNldC5wdWxsKG1heF9yZWNvcmRzJTNEMTAwKSUwQSUwQSUyMyUyMHRyYW5zZm9ybSUyMEFyZ2lsbGElMjBkYXRhc2V0JTIwdG8lMjBIRiUyMGRhdGFzZXQlMEFoZl9kYXRhc2V0JTIwJTNEJTIwbG9jYWxfZGF0YXNldC5mb3JtYXRfYXMoJTIyZGF0YXNldHMlMjIpJTBBJTBBJTIzJTIwVGhpcyUyMEhGJTIwZGF0YXNldCUyMGNhbiUyMHRoZW4lMjBiZSUyMGZvcm1hdHRlZCUyQyUyMHN0b3JlZCUyMGFuZCUyMHByb2Nlc3NlZCUyMGludG8lMjBhbnklMjB0YWJ1bGFyJTIwZGF0YSUyMGZvcm1hdCUwQSUyMyUyMERpc3BsYXklMjB0aGUlMjBhbm5vdGF0ZWQlMjBkYXRhc2V0JTNBJTBBaGZfZGF0YXNldC50b19wYW5kYXMoKQ==",highlighted:`remote_dataset = rg.FeedbackDataset.from_argilla(dataset_argilla_name, workspace=<span class="hljs-string">&quot;admin&quot;</span>)

<span class="hljs-comment"># pull the first N records from the remote dataset</span>
local_dataset = remote_dataset.pull(max_records=<span class="hljs-number">100</span>)

<span class="hljs-comment"># transform Argilla dataset to HF dataset</span>
hf_dataset = local_dataset.format_as(<span class="hljs-string">&quot;datasets&quot;</span>)

<span class="hljs-comment"># This HF dataset can then be formatted, stored and processed into any tabular data format</span>
<span class="hljs-comment"># Display the annotated dataset:</span>
hf_dataset.to_pandas()`,wrap:!1}}),Yt=new J({props:{code:"JTIzJTIwU3RvcmUlMjB0aGUlMjBkYXRhc2V0JTIwbG9jYWxseSUwQWhmX2RhdGFzZXQudG9fY3N2KCUyMmFyZ2lsbGEtZGF0YXNldC1sb2NhbC5jc3YlMjIpJTIwJTIwJTIzJTIwU2F2ZSUyMGFzJTIwQ1NWJTBBJTIzJTIwaGZfZGF0YXNldC50b19qc29uKCUyMmFyZ2lsbGEtZGF0YXNldC1sb2NhbC5qc29uJTIyKSUyMCUyMCUyMyUyMFNhdmUlMjBhcyUyMEpTT04lMEElMjMlMjBoZl9kYXRhc2V0LnNhdmVfdG9fZGlzayglMjJhcmdpbGxhLWRhdGFzZXQtbG9jYWwlMjIpJTIwJTIwJTIzJTIwU2F2ZSUyMGFzJTIwYSUyMCU2MGRhdGFzZXRzLkRhdGFzZXQlNjAlMjBpbiUyMHRoZSUyMGxvY2FsJTIwZmlsZXN5c3RlbSUwQSUyMyUyMGhmX2RhdGFzZXQudG9fcGFycXVldCgpJTIwJTIwJTIzJTIwU2F2ZSUyMGFzJTIwUGFycXVldA==",highlighted:`<span class="hljs-comment"># Store the dataset locally</span>
hf_dataset.to_csv(<span class="hljs-string">&quot;argilla-dataset-local.csv&quot;</span>)  <span class="hljs-comment"># Save as CSV</span>
<span class="hljs-comment"># hf_dataset.to_json(&quot;argilla-dataset-local.json&quot;)  # Save as JSON</span>
<span class="hljs-comment"># hf_dataset.save_to_disk(&quot;argilla-dataset-local&quot;)  # Save as a \`datasets.Dataset\` in the local filesystem</span>
<span class="hljs-comment"># hf_dataset.to_parquet()  # Save as Parquet</span>`,wrap:!1}}),Vt=new h({props:{title:"Next Steps",local:"next-steps",headingTag:"h2"}}),_t=new Is({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/en/enterprise_cookbook_argilla.md"}}),{c(){m=M("meta"),b=a(),U=M("p"),xt=a(),r(f.$$.fragment),Lt=a(),r(g.$$.fragment),Dt=a(),C=M("p"),C.innerHTML=we,Pt=a(),G=M("p"),G.textContent=Te,Kt=a(),B=M("p"),B.innerHTML=Ie,Ot=a(),Z=M("p"),Z.innerHTML=be,tl=a(),A=M("ol"),A.innerHTML=fe,ll=a(),R=M("p"),R.textContent=ge,el=a(),r(k.$$.fragment),sl=a(),r(W.$$.fragment),al=a(),r(v.$$.fragment),nl=a(),r(N.$$.fragment),Ml=a(),X=M("p"),X.innerHTML=Ce,ol=a(),r(Q.$$.fragment),il=a(),Y=M("pre"),Y.textContent=Ge,rl=a(),r(V.$$.fragment),pl=a(),r(E.$$.fragment),cl=a(),F=M("p"),F.innerHTML=Be,yl=a(),r(S.$$.fragment),dl=a(),H=M("pre"),H.innerHTML=Ze,jl=a(),r(_.$$.fragment),ml=a(),x=M("p"),x.innerHTML=Ae,Jl=a(),r(z.$$.fragment),hl=a(),$=M("p"),$.innerHTML=Re,ul=a(),r(w.$$.fragment),Ul=a(),r(q.$$.fragment),wl=a(),L=M("pre"),L.textContent=ke,Tl=a(),r(D.$$.fragment),Il=a(),P=M("p"),P.textContent=We,bl=a(),r(K.$$.fragment),fl=a(),r(O.$$.fragment),gl=a(),tt=M("p"),tt.innerHTML=ve,Cl=a(),lt=M("p"),lt.innerHTML=Ne,Gl=a(),et=M("ol"),et.innerHTML=Xe,Bl=a(),T=M("img"),Zl=a(),st=M("p"),st.textContent=Ye,Al=a(),at=M("table"),at.innerHTML=Ve,Rl=a(),nt=M("p"),nt.innerHTML=Ee,kl=a(),Mt=M("p"),Mt.textContent=Fe,Wl=a(),ot=M("img"),vl=a(),r(it.$$.fragment),Nl=a(),rt=M("p"),rt.textContent=He,Xl=a(),pt=M("p"),pt.innerHTML=_e,Ql=a(),r(ct.$$.fragment),Yl=a(),r(yt.$$.fragment),Vl=a(),r(dt.$$.fragment),El=a(),jt=M("p"),jt.textContent=xe,Fl=a(),mt=M("ul"),mt.innerHTML=ze,Sl=a(),Jt=M("p"),Jt.innerHTML=$e,Hl=a(),r(ht.$$.fragment),_l=a(),ut=M("p"),ut.innerHTML=qe,xl=a(),r(Ut.$$.fragment),zl=a(),wt=M("p"),wt.innerHTML=Le,$l=a(),r(Tt.$$.fragment),ql=a(),It=M("p"),It.innerHTML=De,Ll=a(),bt=M("img"),Dl=a(),r(ft.$$.fragment),Pl=a(),gt=M("p"),gt.innerHTML=Ke,Kl=a(),r(Ct.$$.fragment),Ol=a(),Gt=M("p"),Gt.textContent=Oe,te=a(),r(Bt.$$.fragment),le=a(),Zt=M("p"),Zt.innerHTML=ts,ee=a(),At=M("img"),se=a(),Rt=M("p"),Rt.innerHTML=es,ae=a(),I=M("img"),ne=a(),r(kt.$$.fragment),Me=a(),Wt=M("p"),Wt.textContent=as,oe=a(),vt=M("p"),vt.innerHTML=ns,ie=a(),r(Nt.$$.fragment),re=a(),Xt=M("p"),Xt.innerHTML=Ms,pe=a(),r(Qt.$$.fragment),ce=a(),r(Yt.$$.fragment),ye=a(),r(Vt.$$.fragment),de=a(),Et=M("p"),Et.textContent=os,je=a(),Ft=M("p"),Ft.innerHTML=is,me=a(),St=M("ol"),St.innerHTML=rs,Je=a(),Ht=M("p"),Ht.innerHTML=ps,he=a(),r(_t.$$.fragment),ue=a(),zt=M("p"),this.h()},l(t){const l=us("svelte-u9bgzb",document.head);m=o(l,"META",{name:!0,content:!0}),l.forEach(e),b=n(t),U=o(t,"P",{}),ys(U).forEach(e),xt=n(t),p(f.$$.fragment,t),Lt=n(t),p(g.$$.fragment,t),Dt=n(t),C=o(t,"P",{"data-svelte-h":!0}),i(C)!=="svelte-phy2er"&&(C.innerHTML=we),Pt=n(t),G=o(t,"P",{"data-svelte-h":!0}),i(G)!=="svelte-cy2moq"&&(G.textContent=Te),Kt=n(t),B=o(t,"P",{"data-svelte-h":!0}),i(B)!=="svelte-1u64310"&&(B.innerHTML=Ie),Ot=n(t),Z=o(t,"P",{"data-svelte-h":!0}),i(Z)!=="svelte-1j66clp"&&(Z.innerHTML=be),tl=n(t),A=o(t,"OL",{"data-svelte-h":!0}),i(A)!=="svelte-1cnt373"&&(A.innerHTML=fe),ll=n(t),R=o(t,"P",{"data-svelte-h":!0}),i(R)!=="svelte-167x7n3"&&(R.textContent=ge),el=n(t),p(k.$$.fragment,t),sl=n(t),p(W.$$.fragment,t),al=n(t),p(v.$$.fragment,t),nl=n(t),p(N.$$.fragment,t),Ml=n(t),X=o(t,"P",{"data-svelte-h":!0}),i(X)!=="svelte-1ev16fs"&&(X.innerHTML=Ce),ol=n(t),p(Q.$$.fragment,t),il=n(t),Y=o(t,"PRE",{"data-svelte-h":!0}),i(Y)!=="svelte-ky4em7"&&(Y.textContent=Ge),rl=n(t),p(V.$$.fragment,t),pl=n(t),p(E.$$.fragment,t),cl=n(t),F=o(t,"P",{"data-svelte-h":!0}),i(F)!=="svelte-1i1rnp2"&&(F.innerHTML=Be),yl=n(t),p(S.$$.fragment,t),dl=n(t),H=o(t,"PRE",{"data-svelte-h":!0}),i(H)!=="svelte-dhulxx"&&(H.innerHTML=Ze),jl=n(t),p(_.$$.fragment,t),ml=n(t),x=o(t,"P",{"data-svelte-h":!0}),i(x)!=="svelte-yy0mhz"&&(x.innerHTML=Ae),Jl=n(t),p(z.$$.fragment,t),hl=n(t),$=o(t,"P",{"data-svelte-h":!0}),i($)!=="svelte-1sdd35o"&&($.innerHTML=Re),ul=n(t),p(w.$$.fragment,t),Ul=n(t),p(q.$$.fragment,t),wl=n(t),L=o(t,"PRE",{"data-svelte-h":!0}),i(L)!=="svelte-10h0gq9"&&(L.textContent=ke),Tl=n(t),p(D.$$.fragment,t),Il=n(t),P=o(t,"P",{"data-svelte-h":!0}),i(P)!=="svelte-1rfbvc"&&(P.textContent=We),bl=n(t),p(K.$$.fragment,t),fl=n(t),p(O.$$.fragment,t),gl=n(t),tt=o(t,"P",{"data-svelte-h":!0}),i(tt)!=="svelte-1teqijw"&&(tt.innerHTML=ve),Cl=n(t),lt=o(t,"P",{"data-svelte-h":!0}),i(lt)!=="svelte-fjzpmi"&&(lt.innerHTML=Ne),Gl=n(t),et=o(t,"OL",{"data-svelte-h":!0}),i(et)!=="svelte-1jdfzyi"&&(et.innerHTML=Xe),Bl=n(t),T=o(t,"IMG",{src:!0,width:!0,alt:!0}),Zl=n(t),st=o(t,"P",{"data-svelte-h":!0}),i(st)!=="svelte-kcc6ji"&&(st.textContent=Ye),Al=n(t),at=o(t,"TABLE",{"data-svelte-h":!0}),i(at)!=="svelte-1ugfxjt"&&(at.innerHTML=Ve),Rl=n(t),nt=o(t,"P",{"data-svelte-h":!0}),i(nt)!=="svelte-ph0ues"&&(nt.innerHTML=Ee),kl=n(t),Mt=o(t,"P",{"data-svelte-h":!0}),i(Mt)!=="svelte-h86em0"&&(Mt.textContent=Fe),Wl=n(t),ot=o(t,"IMG",{src:!0,alt:!0}),vl=n(t),p(it.$$.fragment,t),Nl=n(t),rt=o(t,"P",{"data-svelte-h":!0}),i(rt)!=="svelte-1kbxreq"&&(rt.textContent=He),Xl=n(t),pt=o(t,"P",{"data-svelte-h":!0}),i(pt)!=="svelte-okmayf"&&(pt.innerHTML=_e),Ql=n(t),p(ct.$$.fragment,t),Yl=n(t),p(yt.$$.fragment,t),Vl=n(t),p(dt.$$.fragment,t),El=n(t),jt=o(t,"P",{"data-svelte-h":!0}),i(jt)!=="svelte-ezpv3g"&&(jt.textContent=xe),Fl=n(t),mt=o(t,"UL",{"data-svelte-h":!0}),i(mt)!=="svelte-1na5fga"&&(mt.innerHTML=ze),Sl=n(t),Jt=o(t,"P",{"data-svelte-h":!0}),i(Jt)!=="svelte-1qpaxi2"&&(Jt.innerHTML=$e),Hl=n(t),p(ht.$$.fragment,t),_l=n(t),ut=o(t,"P",{"data-svelte-h":!0}),i(ut)!=="svelte-1h73c1v"&&(ut.innerHTML=qe),xl=n(t),p(Ut.$$.fragment,t),zl=n(t),wt=o(t,"P",{"data-svelte-h":!0}),i(wt)!=="svelte-4od12y"&&(wt.innerHTML=Le),$l=n(t),p(Tt.$$.fragment,t),ql=n(t),It=o(t,"P",{"data-svelte-h":!0}),i(It)!=="svelte-1wxrrfl"&&(It.innerHTML=De),Ll=n(t),bt=o(t,"IMG",{src:!0,alt:!0}),Dl=n(t),p(ft.$$.fragment,t),Pl=n(t),gt=o(t,"P",{"data-svelte-h":!0}),i(gt)!=="svelte-1w6terr"&&(gt.innerHTML=Ke),Kl=n(t),p(Ct.$$.fragment,t),Ol=n(t),Gt=o(t,"P",{"data-svelte-h":!0}),i(Gt)!=="svelte-jh7i3q"&&(Gt.textContent=Oe),te=n(t),p(Bt.$$.fragment,t),le=n(t),Zt=o(t,"P",{"data-svelte-h":!0}),i(Zt)!=="svelte-swre3v"&&(Zt.innerHTML=ts),ee=n(t),At=o(t,"IMG",{src:!0,alt:!0}),se=n(t),Rt=o(t,"P",{"data-svelte-h":!0}),i(Rt)!=="svelte-2go05s"&&(Rt.innerHTML=es),ae=n(t),I=o(t,"IMG",{src:!0,width:!0,alt:!0}),ne=n(t),p(kt.$$.fragment,t),Me=n(t),Wt=o(t,"P",{"data-svelte-h":!0}),i(Wt)!=="svelte-1cz4pgm"&&(Wt.textContent=as),oe=n(t),vt=o(t,"P",{"data-svelte-h":!0}),i(vt)!=="svelte-102nyy6"&&(vt.innerHTML=ns),ie=n(t),p(Nt.$$.fragment,t),re=n(t),Xt=o(t,"P",{"data-svelte-h":!0}),i(Xt)!=="svelte-d4qx1v"&&(Xt.innerHTML=Ms),pe=n(t),p(Qt.$$.fragment,t),ce=n(t),p(Yt.$$.fragment,t),ye=n(t),p(Vt.$$.fragment,t),de=n(t),Et=o(t,"P",{"data-svelte-h":!0}),i(Et)!=="svelte-1vu6lzx"&&(Et.textContent=os),je=n(t),Ft=o(t,"P",{"data-svelte-h":!0}),i(Ft)!=="svelte-1nz4tc1"&&(Ft.innerHTML=is),me=n(t),St=o(t,"OL",{"data-svelte-h":!0}),i(St)!=="svelte-1nta8f4"&&(St.innerHTML=rs),Je=n(t),Ht=o(t,"P",{"data-svelte-h":!0}),i(Ht)!=="svelte-afghff"&&(Ht.innerHTML=ps),he=n(t),p(_t.$$.fragment,t),ue=n(t),zt=o(t,"P",{}),ys(zt).forEach(e),this.h()},h(){u(m,"name","hf:doc:metadata"),u(m,"content",gs),$t(T.src,Qe="https://github.com/MoritzLaurer/huggingface_materials/blob/996d4f942948335b884bd251b4d2d5ce0e162468/images_misc/argilla-space-variables.png?raw=true")||u(T,"src",Qe),u(T,"width","350"),u(T,"alt","image description"),$t(ot.src,Se="https://raw.githubusercontent.com/MoritzLaurer/huggingface_materials/996d4f942948335b884bd251b4d2d5ce0e162468/images_misc/argilla-login-screen.png")||u(ot,"src",Se),u(ot,"alt","image description"),$t(bt.src,Pe="https://github.com/MoritzLaurer/huggingface_materials/blob/4d4428d68d8a4a087c4255f6838e261d9d5bfbda/images_misc/argilla-tasks-overview.png?raw=true")||u(bt,"src",Pe),u(bt,"alt","image description"),$t(At.src,ls="https://github.com/MoritzLaurer/huggingface_materials/blob/27ed6b49365bed9e3f65a675c322ec935086479c/images_misc/argilla-codellm-interface.png?raw=true")||u(At,"src",ls),u(At,"alt","image description"),$t(I.src,ss="https://github.com/MoritzLaurer/huggingface_materials/blob/996d4f942948335b884bd251b4d2d5ce0e162468/images_misc/argilla-annotators-metadata.png?raw=true")||u(I,"src",ss),u(I,"width","350"),u(I,"alt","image description")},m(t,l){Us(document.head,m),s(t,b,l),s(t,U,l),s(t,xt,l),c(f,t,l),s(t,Lt,l),c(g,t,l),s(t,Dt,l),s(t,C,l),s(t,Pt,l),s(t,G,l),s(t,Kt,l),s(t,B,l),s(t,Ot,l),s(t,Z,l),s(t,tl,l),s(t,A,l),s(t,ll,l),s(t,R,l),s(t,el,l),c(k,t,l),s(t,sl,l),c(W,t,l),s(t,al,l),c(v,t,l),s(t,nl,l),c(N,t,l),s(t,Ml,l),s(t,X,l),s(t,ol,l),c(Q,t,l),s(t,il,l),s(t,Y,l),s(t,rl,l),c(V,t,l),s(t,pl,l),c(E,t,l),s(t,cl,l),s(t,F,l),s(t,yl,l),c(S,t,l),s(t,dl,l),s(t,H,l),s(t,jl,l),c(_,t,l),s(t,ml,l),s(t,x,l),s(t,Jl,l),c(z,t,l),s(t,hl,l),s(t,$,l),s(t,ul,l),c(w,t,l),s(t,Ul,l),c(q,t,l),s(t,wl,l),s(t,L,l),s(t,Tl,l),c(D,t,l),s(t,Il,l),s(t,P,l),s(t,bl,l),c(K,t,l),s(t,fl,l),c(O,t,l),s(t,gl,l),s(t,tt,l),s(t,Cl,l),s(t,lt,l),s(t,Gl,l),s(t,et,l),s(t,Bl,l),s(t,T,l),s(t,Zl,l),s(t,st,l),s(t,Al,l),s(t,at,l),s(t,Rl,l),s(t,nt,l),s(t,kl,l),s(t,Mt,l),s(t,Wl,l),s(t,ot,l),s(t,vl,l),c(it,t,l),s(t,Nl,l),s(t,rt,l),s(t,Xl,l),s(t,pt,l),s(t,Ql,l),c(ct,t,l),s(t,Yl,l),c(yt,t,l),s(t,Vl,l),c(dt,t,l),s(t,El,l),s(t,jt,l),s(t,Fl,l),s(t,mt,l),s(t,Sl,l),s(t,Jt,l),s(t,Hl,l),c(ht,t,l),s(t,_l,l),s(t,ut,l),s(t,xl,l),c(Ut,t,l),s(t,zl,l),s(t,wt,l),s(t,$l,l),c(Tt,t,l),s(t,ql,l),s(t,It,l),s(t,Ll,l),s(t,bt,l),s(t,Dl,l),c(ft,t,l),s(t,Pl,l),s(t,gt,l),s(t,Kl,l),c(Ct,t,l),s(t,Ol,l),s(t,Gt,l),s(t,te,l),c(Bt,t,l),s(t,le,l),s(t,Zt,l),s(t,ee,l),s(t,At,l),s(t,se,l),s(t,Rt,l),s(t,ae,l),s(t,I,l),s(t,ne,l),c(kt,t,l),s(t,Me,l),s(t,Wt,l),s(t,oe,l),s(t,vt,l),s(t,ie,l),c(Nt,t,l),s(t,re,l),s(t,Xt,l),s(t,pe,l),c(Qt,t,l),s(t,ce,l),c(Yt,t,l),s(t,ye,l),c(Vt,t,l),s(t,de,l),s(t,Et,l),s(t,je,l),s(t,Ft,l),s(t,me,l),s(t,St,l),s(t,Je,l),s(t,Ht,l),s(t,he,l),c(_t,t,l),s(t,ue,l),s(t,zt,l),Ue=!0},p(t,[l]){const cs={};l&2&&(cs.$$scope={dirty:l,ctx:t}),w.$set(cs)},i(t){Ue||(y(f.$$.fragment,t),y(g.$$.fragment,t),y(k.$$.fragment,t),y(W.$$.fragment,t),y(v.$$.fragment,t),y(N.$$.fragment,t),y(Q.$$.fragment,t),y(V.$$.fragment,t),y(E.$$.fragment,t),y(S.$$.fragment,t),y(_.$$.fragment,t),y(z.$$.fragment,t),y(w.$$.fragment,t),y(q.$$.fragment,t),y(D.$$.fragment,t),y(K.$$.fragment,t),y(O.$$.fragment,t),y(it.$$.fragment,t),y(ct.$$.fragment,t),y(yt.$$.fragment,t),y(dt.$$.fragment,t),y(ht.$$.fragment,t),y(Ut.$$.fragment,t),y(Tt.$$.fragment,t),y(ft.$$.fragment,t),y(Ct.$$.fragment,t),y(Bt.$$.fragment,t),y(kt.$$.fragment,t),y(Nt.$$.fragment,t),y(Qt.$$.fragment,t),y(Yt.$$.fragment,t),y(Vt.$$.fragment,t),y(_t.$$.fragment,t),Ue=!0)},o(t){d(f.$$.fragment,t),d(g.$$.fragment,t),d(k.$$.fragment,t),d(W.$$.fragment,t),d(v.$$.fragment,t),d(N.$$.fragment,t),d(Q.$$.fragment,t),d(V.$$.fragment,t),d(E.$$.fragment,t),d(S.$$.fragment,t),d(_.$$.fragment,t),d(z.$$.fragment,t),d(w.$$.fragment,t),d(q.$$.fragment,t),d(D.$$.fragment,t),d(K.$$.fragment,t),d(O.$$.fragment,t),d(it.$$.fragment,t),d(ct.$$.fragment,t),d(yt.$$.fragment,t),d(dt.$$.fragment,t),d(ht.$$.fragment,t),d(Ut.$$.fragment,t),d(Tt.$$.fragment,t),d(ft.$$.fragment,t),d(Ct.$$.fragment,t),d(Bt.$$.fragment,t),d(kt.$$.fragment,t),d(Nt.$$.fragment,t),d(Qt.$$.fragment,t),d(Yt.$$.fragment,t),d(Vt.$$.fragment,t),d(_t.$$.fragment,t),Ue=!1},d(t){t&&(e(b),e(U),e(xt),e(Lt),e(Dt),e(C),e(Pt),e(G),e(Kt),e(B),e(Ot),e(Z),e(tl),e(A),e(ll),e(R),e(el),e(sl),e(al),e(nl),e(Ml),e(X),e(ol),e(il),e(Y),e(rl),e(pl),e(cl),e(F),e(yl),e(dl),e(H),e(jl),e(ml),e(x),e(Jl),e(hl),e($),e(ul),e(Ul),e(wl),e(L),e(Tl),e(Il),e(P),e(bl),e(fl),e(gl),e(tt),e(Cl),e(lt),e(Gl),e(et),e(Bl),e(T),e(Zl),e(st),e(Al),e(at),e(Rl),e(nt),e(kl),e(Mt),e(Wl),e(ot),e(vl),e(Nl),e(rt),e(Xl),e(pt),e(Ql),e(Yl),e(Vl),e(El),e(jt),e(Fl),e(mt),e(Sl),e(Jt),e(Hl),e(_l),e(ut),e(xl),e(zl),e(wt),e($l),e(ql),e(It),e(Ll),e(bt),e(Dl),e(Pl),e(gt),e(Kl),e(Ol),e(Gt),e(te),e(le),e(Zt),e(ee),e(At),e(se),e(Rt),e(ae),e(I),e(ne),e(Me),e(Wt),e(oe),e(vt),e(ie),e(re),e(Xt),e(pe),e(ce),e(ye),e(de),e(Et),e(je),e(Ft),e(me),e(St),e(Je),e(Ht),e(he),e(ue),e(zt)),e(m),j(f,t),j(g,t),j(k,t),j(W,t),j(v,t),j(N,t),j(Q,t),j(V,t),j(E,t),j(S,t),j(_,t),j(z,t),j(w,t),j(q,t),j(D,t),j(K,t),j(O,t),j(it,t),j(ct,t),j(yt,t),j(dt,t),j(ht,t),j(Ut,t),j(Tt,t),j(ft,t),j(Ct,t),j(Bt,t),j(kt,t),j(Nt,t),j(Qt,t),j(Yt,t),j(Vt,t),j(_t,t)}}}const gs='{"title":"Data Annotation with Argilla Spaces","local":"data-annotation-with-argilla-spaces","sections":[{"title":"Install required packages and connect to HF Hub","local":"install-required-packages-and-connect-to-hf-hub","sections":[],"depth":2},{"title":"Download example task data","local":"download-example-task-data","sections":[],"depth":2},{"title":"Prompt two LLMs on the example task","local":"prompt-two-llms-on-the-example-task","sections":[{"title":"Formatting the instructions with a chat_template","local":"formatting-the-instructions-with-a-chattemplate","sections":[],"depth":4},{"title":"Sending the instructions to the HF Inference API","local":"sending-the-instructions-to-the-hf-inference-api","sections":[],"depth":4},{"title":"Store the LLM outputs in a dataset","local":"store-the-llm-outputs-in-a-dataset","sections":[],"depth":4}],"depth":2},{"title":"Create and configure your Argilla annotation interface","local":"create-and-configure-your-argilla-annotation-interface","sections":[{"title":"Programmatically interact with Argilla","local":"programmatically-interact-with-argilla","sections":[],"depth":4},{"title":"Write good annotator guidelines","local":"write-good-annotator-guidelines","sections":[],"depth":4},{"title":"Tailor the Argilla interface to your specific task","local":"tailor-the-argilla-interface-to-your-specific-task","sections":[],"depth":4},{"title":"Upload data to Argilla for our task","local":"upload-data-to-argilla-for-our-task","sections":[],"depth":4}],"depth":2},{"title":"Annotate","local":"annotate","sections":[],"depth":2},{"title":"Download annotated data","local":"download-annotated-data","sections":[],"depth":2},{"title":"Next Steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function Cs(qt){return js(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ks extends Js{constructor(m){super(),hs(this,m,Cs,fs,ds,{})}}export{ks as component};
