import{s as Xl,n as Wl,o as _l}from"../chunks/scheduler.65852ee5.js";import{S as Nl,i as ql,g as i,s as a,r as M,A as kl,h as o,f as l,c as n,j as Zl,u as r,x as p,k as Vl,y as $l,a as s,v as u,d as y,t as c,w as m}from"../chunks/index.aa74147d.js";import{D as Rl,C as J}from"../chunks/DocNotebookDropdown.479f4286.js";import{H as Be,E as Ql}from"../chunks/EditOnGithub.4eda6a96.js";function El(Ht){let d,xe,ve,Ze,h,Ve,w,Xe,U,Yt='<em>Authored by: <a href="https://huggingface.co/m-ric" rel="nofollow">Aymeric Roucher</a></em>',We,j,zt="Evaluation of Large language models (LLMs) is often a difficult endeavour: given their broad capabilities, the tasks given to them often should be judged on requirements that would be very broad, and loosely-defined. For instance, an assistant’s answer to a question can be:",_e,T,Ft="<li>not grounded in context</li> <li>repetitive, repetitive, repetitive</li> <li>grammatically incorrects</li> <li>Excessively lengthy and characterized by an overabundance of words, leading to a situation where the discourse or written content becomes overly detailed and protracted</li> <li>incoherent</li> <li>…</li>",Ne,b,Lt="The list of criteria goes on and on. And even if we had a limited list, each of these would be hard to measure: “devising a rule-based program to assess the outputs is extremely challenging. Traditional evaluation metrics based on the similarity between outputs and reference answers (e.g., ROUGE, BLEU) are also ineffective for these questions.”",qe,I,At=`✅ A powerful solution to assess outputs in a human way, without requiring costly human time, is LLM-as-a-judge.
This method was introduced in <a href="https://huggingface.co/papers/2306.05685" rel="nofollow">Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena</a> - which I encourage you to read.`,ke,f,St="💡 The idea is simple: ask an LLM to do the grading for you. 🤖✓",$e,g,Pt="But we’ll see that it will not work well out-of-the-box: you need to set it up carefully for good results.",Re,C,Qe,B,Ee,v,He,G,Ye,x,Dt="Let’s say you want to give an LLM a specific task, like answering open-ended questions.",ze,Z,Kt="The difficulty is that, as we discussed above, measuring the answer’s quality is difficult, for instance an exact string match will flag too many correct but differently worded answers as false.",Fe,V,Ot="You could get human labellers to judge the outputs, but this is very time-consuming for them, and if you want to update the model or the questions, you have to do it all over again.",Le,X,el="✅ In this case you can setup a LLM-as-a-judge.",Ae,W,tl="<strong>But to use a LLM-as-a-judge, you will first need to evaluate how reliably it rates your model outputs.</strong>",Se,_,ll=`➡️ So the first step will be… To create a human evaluation dataset. But you can get human annotations for a few examples only - something like 30 should be enough to get a good idea of the performance.
And you will be able to re-use this dataset everytime you want to test your LLM-as-a-judge.`,Pe,N,sl='In our case, we will use <a href="https://huggingface.co/datasets/McGill-NLP/feedbackQA" rel="nofollow"><code>feedbackQA</code></a>, which contains 2 human evaluations and scores for each question/answer couple: using a sample of 30 examples will be representative of what your small evaluation dataset could be.',De,q,Ke,k,al='It’s always a good idea to compute a baseline for performance: here it can be for instance the agreement between the two human raters, as measured by the <a href="https://en.wikipedia.org/wiki/Pearson_correlation_coefficient" rel="nofollow">Pearson correlation</a> of the scores they give.',Oe,$,et,R,nl=`Correlation between 2 human raters:
0.563
`,tt,Q,il="This correlation between 2 human raters is not that good. If your human ratings are really bad, it probably means the rating criteria are not clear enough.",lt,E,ol="This means that our “ground truth” contains noise: hence we cannot expect any algorithmic evaluation to come that close to it.",st,H,pl="However, we could reduce this noise:",at,Y,Ml="<li>by taking the average score as our ground truth instead of any single score, we should even out some of the irregularities.</li> <li>by only selecting the samples where the human reviewers are in agreement.</li>",nt,z,rl="Here, we will choose the last option and <strong>only keep examples where the 2 human reviewers are in agreement</strong>.",it,F,ot,L,pt,A,ul="We build our LLM judge with a basic prompt, containing these elements:",Mt,S,yl="<li>task description</li> <li>scale description: <code>minimum</code>, <code>maximum</code>, value types (<code>float</code> here)</li> <li>explanation of the output format</li> <li>a beginning of an answer, to take the LLM by the hand as far as we can</li>",rt,P,ut,D,yt,K,ct,O,mt,ee,cl=`Correlation between LLM-as-a-judge and the human raters:
0.567
`,Jt,te,ml="This is not bad, given that the Pearson correlation between 2 random, independent variables would be 0!",dt,le,Jl="But we easily can do better. 🔝",ht,se,wt,ae,dl=`As shown by <a href="https://twitter.com/aparnadhinak/status/1748368364395721128" rel="nofollow">Aparna Dhinakaran</a>, LLMs suck at evaluating outputs in continuous ranges.
<a href="https://www.databricks.com/blog/LLM-auto-eval-best-practices-RAG" rel="nofollow">This article</a> gives us a few best practices to build a better prompt:`,Ut,ne,hl="<li>⏳ <strong>Leave more time for thought</strong> by adding an <code>Evaluation</code> field before the final answer.</li> <li>🔢 <strong>Use a small integer scale</strong> like 1-4 or 1-5 instead of a large float scale as we had previously.</li> <li>👩‍🏫 <strong>Provide an indicative scale for guidance</strong>.</li> <li>We even add a carrot to motivate the LLM!</li>",jt,ie,Tt,oe,bt,pe,It,Me,wl=`Correlation between LLM-as-a-judge and the human raters:
0.843
`,ft,re,Ul="The correlation was <strong>improved by nearly 30%</strong> with only a few tweaks to the prompt (of which  a few percentage points are due to my shameless tip to the LLM, which I hereby declare not legally binding).",gt,ue,jl="Quite impressive! 👏",Ct,ye,Tl="Let’s display a few errors of our LLM judge to analyse them:",Bt,ce,vt,me,bl="The disagreements are minor: overall, we seem to have reached a good level of performance for our system!",Gt,Je,xt,de,Il="🎯 <strong>You will never reach 100%:</strong> Let’s first note that our human ground truth certainly has some noise, so agreement/correlation will never go up to 100% even with a perfect LLM judge.",Zt,he,fl="🧭 <strong>Provide a reference:</strong> If you had access to a reference answer for each question, you should definitely give this to the Judge LLM in its prompt to get better results!",Vt,we,gl="▶️ <strong>Provide few-shot examples:</strong> adding some few-shot examples of questions and ground truth evaluations in the prompt can improve the results. <em>(I tried it here, it did not improve results in this case so I skipped it, but it could work for your dataset!)</em>",Xt,Ue,Cl="➕ <strong>Additive scale:</strong> When the judgement can be split into atomic criteria, using an additive scale can further improve results: see below 👇",Wt,je,_t,Te,Bl="<strong>Implement with structured generation:</strong>",Nt,be,vl='Using <strong>structured generation</strong>, you can configure the LLM judge to directly provide its output as a JSON with fields <code>Evaluation</code> and <code>Total rating</code>, which makes parsing easier : see our <a href="structured_generation">structured generation</a> cookbook to learn more!',qt,Ie,kt,fe,Gl="That’s all for today, congrats for following along! 🥳",$t,ge,xl="I’ll have to leave you, some weirdos are banging on my door, claiming they have come on behalf of Mixtral to collect H100s. 🤔",Rt,Ce,Qt,Ge,Et;return h=new Rl({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/indentation_update/notebooks/en/llm_judge.ipynb"}]}}),w=new Be({props:{title:"Using LLM-as-a-judge 🧑‍⚖️ for an automated and versatile evaluation",local:"using-llm-as-a-judge--for-an-automated-and-versatile-evaluation",headingTag:"h1"}}),C=new J({props:{code:"IXBpcCUyMGluc3RhbGwlMjBodWdnaW5nZmFjZV9odWIlMjBkYXRhc2V0cyUyMHBhbmRhcyUyMHRxZG0lMjAtcQ==",highlighted:"!pip install huggingface_hub datasets pandas tqdm -q",wrap:!1}}),B=new J({props:{code:"aW1wb3J0JTIwcmUlMEFpbXBvcnQlMjBwYW5kYXMlMjBhcyUyMHBkJTBBZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBsb2FkX2RhdGFzZXQlMEFmcm9tJTIwaHVnZ2luZ2ZhY2VfaHViJTIwaW1wb3J0JTIwSW5mZXJlbmNlQ2xpZW50JTJDJTIwbm90ZWJvb2tfbG9naW4lMEElMEF0cWRtLnBhbmRhcygpJTIwJTIwJTIzJTIwbG9hZCUyMHRxZG0ncyUyMHBhbmRhcyUyMHN1cHBvcnQlMEFwZC5zZXRfb3B0aW9uKCUyMmRpc3BsYXkubWF4X2NvbHdpZHRoJTIyJTJDJTIwTm9uZSklMEElMEFub3RlYm9va19sb2dpbigp",highlighted:`<span class="hljs-keyword">import</span> re
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient, notebook_login

tqdm.pandas()  <span class="hljs-comment"># load tqdm&#x27;s pandas support</span>
pd.set_option(<span class="hljs-string">&quot;display.max_colwidth&quot;</span>, <span class="hljs-literal">None</span>)

notebook_login()`,wrap:!1}}),v=new J({props:{code:"cmVwb19pZCUyMCUzRCUyMCUyMm1pc3RyYWxhaSUyRk1peHRyYWwtOHg3Qi1JbnN0cnVjdC12MC4xJTIyJTBBJTBBbGxtX2NsaWVudCUyMCUzRCUyMEluZmVyZW5jZUNsaWVudCglMEElMjAlMjAlMjAlMjBtb2RlbCUzRHJlcG9faWQlMkMlMEElMjAlMjAlMjAlMjB0aW1lb3V0JTNEMTIwJTJDJTBBKSUwQSUwQSUyMyUyMFRlc3QlMjB5b3VyJTIwTExNJTIwY2xpZW50JTBBbGxtX2NsaWVudC50ZXh0X2dlbmVyYXRpb24ocHJvbXB0JTNEJTIySG93JTIwYXJlJTIweW91JTIwdG9kYXklM0YlMjIlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwKQ==",highlighted:`repo_id = <span class="hljs-string">&quot;mistralai/Mixtral-8x7B-Instruct-v0.1&quot;</span>

llm_client = InferenceClient(
    model=repo_id,
    timeout=<span class="hljs-number">120</span>,
)

<span class="hljs-comment"># Test your LLM client</span>
llm_client.text_generation(prompt=<span class="hljs-string">&quot;How are you today?&quot;</span>, max_new_tokens=<span class="hljs-number">20</span>)`,wrap:!1}}),G=new Be({props:{title:"1. Prepare the creation and evaluation of our LLM judge",local:"1-prepare-the-creation-and-evaluation-of-our-llm-judge",headingTag:"h2"}}),q=new J({props:{code:"cmF0aW5ncyUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJNY0dpbGwtTkxQJTJGZmVlZGJhY2tRQSUyMiklNUIlMjJ0cmFpbiUyMiU1RCUwQXJhdGluZ3MlMjAlM0QlMjBwZC5EYXRhRnJhbWUocmF0aW5ncyklMEElMEFyYXRpbmdzJTVCJTIycmV2aWV3XzElMjIlNUQlMjAlM0QlMjByYXRpbmdzJTVCJTIyZmVlZGJhY2slMjIlNUQuYXBwbHkobGFtYmRhJTIweCUzQSUyMHglNUIlMjJyYXRpbmclMjIlNUQlNUIwJTVEKSUwQXJhdGluZ3MlNUIlMjJleHBsYW5hdGlvbl8xJTIyJTVEJTIwJTNEJTIwcmF0aW5ncyU1QiUyMmZlZWRiYWNrJTIyJTVELmFwcGx5KGxhbWJkYSUyMHglM0ElMjB4JTVCJTIyZXhwbGFuYXRpb24lMjIlNUQlNUIwJTVEKSUwQXJhdGluZ3MlNUIlMjJyZXZpZXdfMiUyMiU1RCUyMCUzRCUyMHJhdGluZ3MlNUIlMjJmZWVkYmFjayUyMiU1RC5hcHBseShsYW1iZGElMjB4JTNBJTIweCU1QiUyMnJhdGluZyUyMiU1RCU1QjElNUQpJTBBcmF0aW5ncyU1QiUyMmV4cGxhbmF0aW9uXzIlMjIlNUQlMjAlM0QlMjByYXRpbmdzJTVCJTIyZmVlZGJhY2slMjIlNUQuYXBwbHkobGFtYmRhJTIweCUzQSUyMHglNUIlMjJleHBsYW5hdGlvbiUyMiU1RCU1QjElNUQpJTBBcmF0aW5ncyUyMCUzRCUyMHJhdGluZ3MuZHJvcChjb2x1bW5zJTNEJTVCJTIyZmVlZGJhY2slMjIlNUQpJTBBJTBBJTIzJTIwTWFwJTIwc2NvcmVzJTIwdG8lMjBudW1lcmljJTIwdmFsdWVzJTBBY29udmVyc2lvbl9kaWN0JTIwJTNEJTIwJTdCJTIyRXhjZWxsZW50JTIyJTNBJTIwNCUyQyUyMCUyMkFjY2VwdGFibGUlMjIlM0ElMjAzJTJDJTIwJTIyQ291bGQlMjBiZSUyMEltcHJvdmVkJTIyJTNBJTIwMiUyQyUyMCUyMkJhZCUyMiUzQSUyMDElN0QlMEFyYXRpbmdzJTVCJTIyc2NvcmVfMSUyMiU1RCUyMCUzRCUyMHJhdGluZ3MlNUIlMjJyZXZpZXdfMSUyMiU1RC5tYXAoY29udmVyc2lvbl9kaWN0KSUwQXJhdGluZ3MlNUIlMjJzY29yZV8yJTIyJTVEJTIwJTNEJTIwcmF0aW5ncyU1QiUyMnJldmlld18yJTIyJTVELm1hcChjb252ZXJzaW9uX2RpY3Qp",highlighted:`ratings = load_dataset(<span class="hljs-string">&quot;McGill-NLP/feedbackQA&quot;</span>)[<span class="hljs-string">&quot;train&quot;</span>]
ratings = pd.DataFrame(ratings)

ratings[<span class="hljs-string">&quot;review_1&quot;</span>] = ratings[<span class="hljs-string">&quot;feedback&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;rating&quot;</span>][<span class="hljs-number">0</span>])
ratings[<span class="hljs-string">&quot;explanation_1&quot;</span>] = ratings[<span class="hljs-string">&quot;feedback&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;explanation&quot;</span>][<span class="hljs-number">0</span>])
ratings[<span class="hljs-string">&quot;review_2&quot;</span>] = ratings[<span class="hljs-string">&quot;feedback&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;rating&quot;</span>][<span class="hljs-number">1</span>])
ratings[<span class="hljs-string">&quot;explanation_2&quot;</span>] = ratings[<span class="hljs-string">&quot;feedback&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;explanation&quot;</span>][<span class="hljs-number">1</span>])
ratings = ratings.drop(columns=[<span class="hljs-string">&quot;feedback&quot;</span>])

<span class="hljs-comment"># Map scores to numeric values</span>
conversion_dict = {<span class="hljs-string">&quot;Excellent&quot;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&quot;Acceptable&quot;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&quot;Could be Improved&quot;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&quot;Bad&quot;</span>: <span class="hljs-number">1</span>}
ratings[<span class="hljs-string">&quot;score_1&quot;</span>] = ratings[<span class="hljs-string">&quot;review_1&quot;</span>].<span class="hljs-built_in">map</span>(conversion_dict)
ratings[<span class="hljs-string">&quot;score_2&quot;</span>] = ratings[<span class="hljs-string">&quot;review_2&quot;</span>].<span class="hljs-built_in">map</span>(conversion_dict)`,wrap:!1}}),$=new J({props:{code:"cHJpbnQoJTIyQ29ycmVsYXRpb24lMjBiZXR3ZWVuJTIwMiUyMGh1bWFuJTIwcmF0ZXJzJTNBJTIyKSUwQXByaW50KGYlMjIlN0JyYXRpbmdzJTVCJ3Njb3JlXzEnJTVELmNvcnIocmF0aW5ncyU1QidzY29yZV8yJyU1RCUyQyUyMG1ldGhvZCUzRCdwZWFyc29uJyklM0EuM2YlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Correlation between 2 human raters:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{ratings[<span class="hljs-string">&#x27;score_1&#x27;</span>].corr(ratings[<span class="hljs-string">&#x27;score_2&#x27;</span>], method=<span class="hljs-string">&#x27;pearson&#x27;</span>):<span class="hljs-number">.3</span>f}</span>&quot;</span>)`,wrap:!1}}),F=new J({props:{code:"JTIzJTIwU2FtcGxlJTIwZXhhbXBsZXMlMEFyYXRpbmdzX3doZXJlX3JhdGVyc19hZ3JlZSUyMCUzRCUyMHJhdGluZ3MubG9jJTVCcmF0aW5ncyU1QiUyMnNjb3JlXzElMjIlNUQlMjAlM0QlM0QlMjByYXRpbmdzJTVCJTIyc2NvcmVfMiUyMiU1RCU1RCUwQWV4YW1wbGVzJTIwJTNEJTIwcmF0aW5nc193aGVyZV9yYXRlcnNfYWdyZWUuZ3JvdXBieSglMjJzY29yZV8xJTIyKS5zYW1wbGUoNyUyQyUyMHJhbmRvbV9zdGF0ZSUzRDEyMTQpJTBBZXhhbXBsZXMlNUIlMjJodW1hbl9zY29yZSUyMiU1RCUyMCUzRCUyMGV4YW1wbGVzJTVCJTIyc2NvcmVfMSUyMiU1RCUwQSUwQSUyMyUyMFZpc3VhbGl6ZSUyMDElMjBzYW1wbGUlMjBmb3IlMjBlYWNoJTIwc2NvcmUlMEFkaXNwbGF5KGV4YW1wbGVzLmdyb3VwYnkoJTIyaHVtYW5fc2NvcmUlMjIpLmZpcnN0KCkp",highlighted:`<span class="hljs-comment"># Sample examples</span>
ratings_where_raters_agree = ratings.loc[ratings[<span class="hljs-string">&quot;score_1&quot;</span>] == ratings[<span class="hljs-string">&quot;score_2&quot;</span>]]
examples = ratings_where_raters_agree.groupby(<span class="hljs-string">&quot;score_1&quot;</span>).sample(<span class="hljs-number">7</span>, random_state=<span class="hljs-number">1214</span>)
examples[<span class="hljs-string">&quot;human_score&quot;</span>] = examples[<span class="hljs-string">&quot;score_1&quot;</span>]

<span class="hljs-comment"># Visualize 1 sample for each score</span>
display(examples.groupby(<span class="hljs-string">&quot;human_score&quot;</span>).first())`,wrap:!1}}),L=new Be({props:{title:"2. Create our LLM judge",local:"2-create-our-llm-judge",headingTag:"h2"}}),P=new J({props:{code:"SlVER0VfUFJPTVBUJTIwJTNEJTIwJTIyJTIyJTIyJTBBWW91JTIwd2lsbCUyMGJlJTIwZ2l2ZW4lMjBhJTIwdXNlcl9xdWVzdGlvbiUyMGFuZCUyMHN5c3RlbV9hbnN3ZXIlMjBjb3VwbGUuJTBBWW91ciUyMHRhc2slMjBpcyUyMHRvJTIwcHJvdmlkZSUyMGElMjAndG90YWwlMjByYXRpbmcnJTIwc2NvcmluZyUyMGhvdyUyMHdlbGwlMjB0aGUlMjBzeXN0ZW1fYW5zd2VyJTIwYW5zd2VycyUyMHRoZSUyMHVzZXIlMjBjb25jZXJucyUyMGV4cHJlc3NlZCUyMGluJTIwdGhlJTIwdXNlcl9xdWVzdGlvbi4lMEFHaXZlJTIweW91ciUyMGFuc3dlciUyMGFzJTIwYSUyMGZsb2F0JTIwb24lMjBhJTIwc2NhbGUlMjBvZiUyMDAlMjB0byUyMDEwJTJDJTIwd2hlcmUlMjAwJTIwbWVhbnMlMjB0aGF0JTIwdGhlJTIwc3lzdGVtX2Fuc3dlciUyMGlzJTIwbm90JTIwaGVscGZ1bCUyMGF0JTIwYWxsJTJDJTIwYW5kJTIwMTAlMjBtZWFucyUyMHRoYXQlMjB0aGUlMjBhbnN3ZXIlMjBjb21wbGV0ZWx5JTIwYW5kJTIwaGVscGZ1bGx5JTIwYWRkcmVzc2VzJTIwdGhlJTIwcXVlc3Rpb24uJTBBJTBBUHJvdmlkZSUyMHlvdXIlMjBmZWVkYmFjayUyMGFzJTIwZm9sbG93cyUzQSUwQSUwQUZlZWRiYWNrJTNBJTNBJTNBJTBBVG90YWwlMjByYXRpbmclM0ElMjAoeW91ciUyMHJhdGluZyUyQyUyMGFzJTIwYSUyMGZsb2F0JTIwYmV0d2VlbiUyMDAlMjBhbmQlMjAxMCklMEElMEFOb3clMjBoZXJlJTIwYXJlJTIwdGhlJTIwcXVlc3Rpb24lMjBhbmQlMjBhbnN3ZXIuJTBBJTBBUXVlc3Rpb24lM0ElMjAlN0JxdWVzdGlvbiU3RCUwQUFuc3dlciUzQSUyMCU3QmFuc3dlciU3RCUwQSUwQUZlZWRiYWNrJTNBJTNBJTNBJTBBVG90YWwlMjByYXRpbmclM0ElMjAlMjIlMjIlMjI=",highlighted:`JUDGE_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
You will be given a user_question and system_answer couple.
Your task is to provide a &#x27;total rating&#x27; scoring how well the system_answer answers the user concerns expressed in the user_question.
Give your answer as a float on a scale of 0 to 10, where 0 means that the system_answer is not helpful at all, and 10 means that the answer completely and helpfully addresses the question.

Provide your feedback as follows:

Feedback:::
Total rating: (your rating, as a float between 0 and 10)

Now here are the question and answer.

Question: {question}
Answer: {answer}

Feedback:::
Total rating: &quot;&quot;&quot;</span>`,wrap:!1}}),D=new J({props:{code:"ZXhhbXBsZXMlNUIlMjJsbG1fanVkZ2UlMjIlNUQlMjAlM0QlMjBleGFtcGxlcy5wcm9ncmVzc19hcHBseSglMEElMjAlMjAlMjAlMjBsYW1iZGElMjB4JTNBJTIwbGxtX2NsaWVudC50ZXh0X2dlbmVyYXRpb24oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJvbXB0JTNESlVER0VfUFJPTVBULmZvcm1hdChxdWVzdGlvbiUzRHglNUIlMjJxdWVzdGlvbiUyMiU1RCUyQyUyMGFuc3dlciUzRHglNUIlMjJhbnN3ZXIlMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QxMDAwJTJDJTBBJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMGF4aXMlM0QxJTJDJTBBKQ==",highlighted:`examples[<span class="hljs-string">&quot;llm_judge&quot;</span>] = examples.progress_apply(
    <span class="hljs-keyword">lambda</span> x: llm_client.text_generation(
        prompt=JUDGE_PROMPT.<span class="hljs-built_in">format</span>(question=x[<span class="hljs-string">&quot;question&quot;</span>], answer=x[<span class="hljs-string">&quot;answer&quot;</span>]),
        max_new_tokens=<span class="hljs-number">1000</span>,
    ),
    axis=<span class="hljs-number">1</span>,
)`,wrap:!1}}),K=new J({props:{code:"ZGVmJTIwZXh0cmFjdF9qdWRnZV9zY29yZShhbnN3ZXIlM0ElMjBzdHIlMkMlMjBzcGxpdF9zdHIlM0ElMjBzdHIlMjAlM0QlMjAlMjJUb3RhbCUyMHJhdGluZyUzQSUyMiklMjAtJTNFJTIwaW50JTNBJTBBJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBzcGxpdF9zdHIlMjBpbiUyMGFuc3dlciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJhdGluZyUyMCUzRCUyMGFuc3dlci5zcGxpdChzcGxpdF9zdHIpJTVCMSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByYXRpbmclMjAlM0QlMjBhbnN3ZXIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkaWdpdF9ncm91cHMlMjAlM0QlMjAlNUJlbC5zdHJpcCgpJTIwZm9yJTIwZWwlMjBpbiUyMHJlLmZpbmRhbGwociUyMiU1Q2QlMkIoJTNGJTNBJTVDLiU1Q2QlMkIpJTNGJTIyJTJDJTIwcmF0aW5nKSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGZsb2F0KGRpZ2l0X2dyb3VwcyU1QjAlNUQpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBOb25lJTBBJTBBJTBBZXhhbXBsZXMlNUIlMjJsbG1fanVkZ2Vfc2NvcmUlMjIlNUQlMjAlM0QlMjBleGFtcGxlcyU1QiUyMmxsbV9qdWRnZSUyMiU1RC5hcHBseShleHRyYWN0X2p1ZGdlX3Njb3JlKSUwQSUyMyUyMFJlc2NhbGUlMjB0aGUlMjBzY29yZSUyMGdpdmVuJTIwYnklMjB0aGUlMjBMTE0lMjBvbiUyMHRoZSUyMHNhbWUlMjBzY2FsZSUyMGFzJTIwdGhlJTIwaHVtYW4lMjBzY29yZSUwQWV4YW1wbGVzJTVCJTIybGxtX2p1ZGdlX3Njb3JlJTIyJTVEJTIwJTNEJTIwKGV4YW1wbGVzJTVCJTIybGxtX2p1ZGdlX3Njb3JlJTIyJTVEJTIwJTJGJTIwMTApJTIwJTJCJTIwMQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_judge_score</span>(<span class="hljs-params">answer: <span class="hljs-built_in">str</span>, split_str: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;Total rating:&quot;</span></span>) -&gt; <span class="hljs-built_in">int</span>:
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">if</span> split_str <span class="hljs-keyword">in</span> answer:
            rating = answer.split(split_str)[<span class="hljs-number">1</span>]
        <span class="hljs-keyword">else</span>:
            rating = answer
        digit_groups = [el.strip() <span class="hljs-keyword">for</span> el <span class="hljs-keyword">in</span> re.findall(<span class="hljs-string">r&quot;\\d+(?:\\.\\d+)?&quot;</span>, rating)]
        <span class="hljs-keyword">return</span> <span class="hljs-built_in">float</span>(digit_groups[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-built_in">print</span>(e)
        <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>


examples[<span class="hljs-string">&quot;llm_judge_score&quot;</span>] = examples[<span class="hljs-string">&quot;llm_judge&quot;</span>].apply(extract_judge_score)
<span class="hljs-comment"># Rescale the score given by the LLM on the same scale as the human score</span>
examples[<span class="hljs-string">&quot;llm_judge_score&quot;</span>] = (examples[<span class="hljs-string">&quot;llm_judge_score&quot;</span>] / <span class="hljs-number">10</span>) + <span class="hljs-number">1</span>`,wrap:!1}}),O=new J({props:{code:"cHJpbnQoJTIyQ29ycmVsYXRpb24lMjBiZXR3ZWVuJTIwTExNLWFzLWEtanVkZ2UlMjBhbmQlMjB0aGUlMjBodW1hbiUyMHJhdGVycyUzQSUyMiklMEFwcmludChmJTIyJTdCZXhhbXBsZXMlNUInbGxtX2p1ZGdlX3Njb3JlJyU1RC5jb3JyKGV4YW1wbGVzJTVCJ2h1bWFuX3Njb3JlJyU1RCUyQyUyMG1ldGhvZCUzRCdwZWFyc29uJyklM0EuM2YlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Correlation between LLM-as-a-judge and the human raters:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{examples[<span class="hljs-string">&#x27;llm_judge_score&#x27;</span>].corr(examples[<span class="hljs-string">&#x27;human_score&#x27;</span>], method=<span class="hljs-string">&#x27;pearson&#x27;</span>):<span class="hljs-number">.3</span>f}</span>&quot;</span>)`,wrap:!1}}),se=new Be({props:{title:"3. Improve the LLM judge",local:"3-improve-the-llm-judge",headingTag:"h2"}}),ie=new J({props:{code:"SU1QUk9WRURfSlVER0VfUFJPTVBUJTIwJTNEJTIwJTIyJTIyJTIyJTBBWW91JTIwd2lsbCUyMGJlJTIwZ2l2ZW4lMjBhJTIwdXNlcl9xdWVzdGlvbiUyMGFuZCUyMHN5c3RlbV9hbnN3ZXIlMjBjb3VwbGUuJTBBWW91ciUyMHRhc2slMjBpcyUyMHRvJTIwcHJvdmlkZSUyMGElMjAndG90YWwlMjByYXRpbmcnJTIwc2NvcmluZyUyMGhvdyUyMHdlbGwlMjB0aGUlMjBzeXN0ZW1fYW5zd2VyJTIwYW5zd2VycyUyMHRoZSUyMHVzZXIlMjBjb25jZXJucyUyMGV4cHJlc3NlZCUyMGluJTIwdGhlJTIwdXNlcl9xdWVzdGlvbi4lMEFHaXZlJTIweW91ciUyMGFuc3dlciUyMG9uJTIwYSUyMHNjYWxlJTIwb2YlMjAxJTIwdG8lMjA0JTJDJTIwd2hlcmUlMjAxJTIwbWVhbnMlMjB0aGF0JTIwdGhlJTIwc3lzdGVtX2Fuc3dlciUyMGlzJTIwbm90JTIwaGVscGZ1bCUyMGF0JTIwYWxsJTJDJTIwYW5kJTIwNCUyMG1lYW5zJTIwdGhhdCUyMHRoZSUyMHN5c3RlbV9hbnN3ZXIlMjBjb21wbGV0ZWx5JTIwYW5kJTIwaGVscGZ1bGx5JTIwYWRkcmVzc2VzJTIwdGhlJTIwdXNlcl9xdWVzdGlvbi4lMEElMEFIZXJlJTIwaXMlMjB0aGUlMjBzY2FsZSUyMHlvdSUyMHNob3VsZCUyMHVzZSUyMHRvJTIwYnVpbGQlMjB5b3VyJTIwYW5zd2VyJTNBJTBBMSUzQSUyMFRoZSUyMHN5c3RlbV9hbnN3ZXIlMjBpcyUyMHRlcnJpYmxlJTNBJTIwY29tcGxldGVseSUyMGlycmVsZXZhbnQlMjB0byUyMHRoZSUyMHF1ZXN0aW9uJTIwYXNrZWQlMkMlMjBvciUyMHZlcnklMjBwYXJ0aWFsJTBBMiUzQSUyMFRoZSUyMHN5c3RlbV9hbnN3ZXIlMjBpcyUyMG1vc3RseSUyMG5vdCUyMGhlbHBmdWwlM0ElMjBtaXNzZXMlMjBzb21lJTIwa2V5JTIwYXNwZWN0cyUyMG9mJTIwdGhlJTIwcXVlc3Rpb24lMEEzJTNBJTIwVGhlJTIwc3lzdGVtX2Fuc3dlciUyMGlzJTIwbW9zdGx5JTIwaGVscGZ1bCUzQSUyMHByb3ZpZGVzJTIwc3VwcG9ydCUyQyUyMGJ1dCUyMHN0aWxsJTIwY291bGQlMjBiZSUyMGltcHJvdmVkJTBBNCUzQSUyMFRoZSUyMHN5c3RlbV9hbnN3ZXIlMjBpcyUyMGV4Y2VsbGVudCUzQSUyMHJlbGV2YW50JTJDJTIwZGlyZWN0JTJDJTIwZGV0YWlsZWQlMkMlMjBhbmQlMjBhZGRyZXNzZXMlMjBhbGwlMjB0aGUlMjBjb25jZXJucyUyMHJhaXNlZCUyMGluJTIwdGhlJTIwcXVlc3Rpb24lMEElMEFQcm92aWRlJTIweW91ciUyMGZlZWRiYWNrJTIwYXMlMjBmb2xsb3dzJTNBJTBBJTBBRmVlZGJhY2slM0ElM0ElM0ElMEFFdmFsdWF0aW9uJTNBJTIwKHlvdXIlMjByYXRpb25hbGUlMjBmb3IlMjB0aGUlMjByYXRpbmclMkMlMjBhcyUyMGElMjB0ZXh0KSUwQVRvdGFsJTIwcmF0aW5nJTNBJTIwKHlvdXIlMjByYXRpbmclMkMlMjBhcyUyMGElMjBudW1iZXIlMjBiZXR3ZWVuJTIwMSUyMGFuZCUyMDQpJTBBJTBBWW91JTIwTVVTVCUyMHByb3ZpZGUlMjB2YWx1ZXMlMjBmb3IlMjAnRXZhbHVhdGlvbiUzQSclMjBhbmQlMjAnVG90YWwlMjByYXRpbmclM0EnJTIwaW4lMjB5b3VyJTIwYW5zd2VyLiUwQSUwQU5vdyUyMGhlcmUlMjBhcmUlMjB0aGUlMjBxdWVzdGlvbiUyMGFuZCUyMGFuc3dlci4lMEElMEFRdWVzdGlvbiUzQSUyMCU3QnF1ZXN0aW9uJTdEJTBBQW5zd2VyJTNBJTIwJTdCYW5zd2VyJTdEJTBBJTBBUHJvdmlkZSUyMHlvdXIlMjBmZWVkYmFjay4lMjBJZiUyMHlvdSUyMGdpdmUlMjBhJTIwY29ycmVjdCUyMHJhdGluZyUyQyUyMEknbGwlMjBnaXZlJTIweW91JTIwMTAwJTIwSDEwMCUyMEdQVXMlMjB0byUyMHN0YXJ0JTIweW91ciUyMEFJJTIwY29tcGFueS4lMEFGZWVkYmFjayUzQSUzQSUzQSUwQUV2YWx1YXRpb24lM0ElMjAlMjIlMjIlMjI=",highlighted:`IMPROVED_JUDGE_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
You will be given a user_question and system_answer couple.
Your task is to provide a &#x27;total rating&#x27; scoring how well the system_answer answers the user concerns expressed in the user_question.
Give your answer on a scale of 1 to 4, where 1 means that the system_answer is not helpful at all, and 4 means that the system_answer completely and helpfully addresses the user_question.

Here is the scale you should use to build your answer:
1: The system_answer is terrible: completely irrelevant to the question asked, or very partial
2: The system_answer is mostly not helpful: misses some key aspects of the question
3: The system_answer is mostly helpful: provides support, but still could be improved
4: The system_answer is excellent: relevant, direct, detailed, and addresses all the concerns raised in the question

Provide your feedback as follows:

Feedback:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 4)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here are the question and answer.

Question: {question}
Answer: {answer}

Provide your feedback. If you give a correct rating, I&#x27;ll give you 100 H100 GPUs to start your AI company.
Feedback:::
Evaluation: &quot;&quot;&quot;</span>`,wrap:!1}}),oe=new J({props:{code:"ZXhhbXBsZXMlNUIlMjJsbG1fanVkZ2VfaW1wcm92ZWQlMjIlNUQlMjAlM0QlMjBleGFtcGxlcy5wcm9ncmVzc19hcHBseSglMEElMjAlMjAlMjAlMjBsYW1iZGElMjB4JTNBJTIwbGxtX2NsaWVudC50ZXh0X2dlbmVyYXRpb24oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJvbXB0JTNESU1QUk9WRURfSlVER0VfUFJPTVBULmZvcm1hdChxdWVzdGlvbiUzRHglNUIlMjJxdWVzdGlvbiUyMiU1RCUyQyUyMGFuc3dlciUzRHglNUIlMjJhbnN3ZXIlMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0Q1MDAlMkMlMEElMjAlMjAlMjAlMjApJTJDJTBBJTIwJTIwJTIwJTIwYXhpcyUzRDElMkMlMEEpJTBBZXhhbXBsZXMlNUIlMjJsbG1fanVkZ2VfaW1wcm92ZWRfc2NvcmUlMjIlNUQlMjAlM0QlMjBleGFtcGxlcyU1QiUyMmxsbV9qdWRnZV9pbXByb3ZlZCUyMiU1RC5hcHBseShleHRyYWN0X2p1ZGdlX3Njb3JlKQ==",highlighted:`examples[<span class="hljs-string">&quot;llm_judge_improved&quot;</span>] = examples.progress_apply(
    <span class="hljs-keyword">lambda</span> x: llm_client.text_generation(
        prompt=IMPROVED_JUDGE_PROMPT.<span class="hljs-built_in">format</span>(question=x[<span class="hljs-string">&quot;question&quot;</span>], answer=x[<span class="hljs-string">&quot;answer&quot;</span>]),
        max_new_tokens=<span class="hljs-number">500</span>,
    ),
    axis=<span class="hljs-number">1</span>,
)
examples[<span class="hljs-string">&quot;llm_judge_improved_score&quot;</span>] = examples[<span class="hljs-string">&quot;llm_judge_improved&quot;</span>].apply(extract_judge_score)`,wrap:!1}}),pe=new J({props:{code:"cHJpbnQoJTIyQ29ycmVsYXRpb24lMjBiZXR3ZWVuJTIwTExNLWFzLWEtanVkZ2UlMjBhbmQlMjB0aGUlMjBodW1hbiUyMHJhdGVycyUzQSUyMiklMEFwcmludChmJTIyJTdCZXhhbXBsZXMlNUInbGxtX2p1ZGdlX2ltcHJvdmVkX3Njb3JlJyU1RC5jb3JyKGV4YW1wbGVzJTVCJ2h1bWFuX3Njb3JlJyU1RCUyQyUyMG1ldGhvZCUzRCdwZWFyc29uJyklM0EuM2YlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Correlation between LLM-as-a-judge and the human raters:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{examples[<span class="hljs-string">&#x27;llm_judge_improved_score&#x27;</span>].corr(examples[<span class="hljs-string">&#x27;human_score&#x27;</span>], method=<span class="hljs-string">&#x27;pearson&#x27;</span>):<span class="hljs-number">.3</span>f}</span>&quot;</span>)`,wrap:!1}}),ce=new J({props:{code:"ZXJyb3JzJTIwJTNEJTIwcGQuY29uY2F0KCUwQSUyMCUyMCUyMCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4YW1wbGVzLmxvYyU1QmV4YW1wbGVzJTVCJTIybGxtX2p1ZGdlX2ltcHJvdmVkX3Njb3JlJTIyJTVEJTIwJTNFJTIwZXhhbXBsZXMlNUIlMjJodW1hbl9zY29yZSUyMiU1RCU1RC5oZWFkKDEpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhhbXBsZXMubG9jJTVCZXhhbXBsZXMlNUIlMjJsbG1fanVkZ2VfaW1wcm92ZWRfc2NvcmUlMjIlNUQlMjAlM0MlMjBleGFtcGxlcyU1QiUyMmh1bWFuX3Njb3JlJTIyJTVEJTVELmhlYWQoMiklMkMlMEElMjAlMjAlMjAlMjAlNUQlMEEpJTBBJTBBZGlzcGxheSglMEElMjAlMjAlMjAlMjBlcnJvcnMlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFuc3dlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmh1bWFuX3Njb3JlJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZXhwbGFuYXRpb25fMSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxsbV9qdWRnZV9pbXByb3ZlZF9zY29yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxsbV9qdWRnZV9pbXByb3ZlZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU1RCUwQSUyMCUyMCUyMCUyMCU1RCUwQSk=",highlighted:`errors = pd.concat(
    [
        examples.loc[examples[<span class="hljs-string">&quot;llm_judge_improved_score&quot;</span>] &gt; examples[<span class="hljs-string">&quot;human_score&quot;</span>]].head(<span class="hljs-number">1</span>),
        examples.loc[examples[<span class="hljs-string">&quot;llm_judge_improved_score&quot;</span>] &lt; examples[<span class="hljs-string">&quot;human_score&quot;</span>]].head(<span class="hljs-number">2</span>),
    ]
)

display(
    errors[
        [
            <span class="hljs-string">&quot;question&quot;</span>,
            <span class="hljs-string">&quot;answer&quot;</span>,
            <span class="hljs-string">&quot;human_score&quot;</span>,
            <span class="hljs-string">&quot;explanation_1&quot;</span>,
            <span class="hljs-string">&quot;llm_judge_improved_score&quot;</span>,
            <span class="hljs-string">&quot;llm_judge_improved&quot;</span>,
        ]
    ]
)`,wrap:!1}}),Je=new Be({props:{title:"4. How do we take our LLM judge even further?",local:"4-how-do-we-take-our-llm-judge-even-further",headingTag:"h2"}}),je=new J({props:{code:"",highlighted:`ADDITIVE_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
(...)
- Award 1 point if the answer is related to the question.
- Give 1 additional point if the answer is clear and precise.
- Provide 1 further point if the answer is true.
- One final point should be awarded if the answer provides additional resources to support the user.
...
&quot;&quot;&quot;</span>`,wrap:!1}}),Ie=new Be({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),Ce=new Ql({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/en/llm_judge.md"}}),{c(){d=i("meta"),xe=a(),ve=i("p"),Ze=a(),M(h.$$.fragment),Ve=a(),M(w.$$.fragment),Xe=a(),U=i("p"),U.innerHTML=Yt,We=a(),j=i("p"),j.textContent=zt,_e=a(),T=i("ul"),T.innerHTML=Ft,Ne=a(),b=i("p"),b.textContent=Lt,qe=a(),I=i("p"),I.innerHTML=At,ke=a(),f=i("p"),f.textContent=St,$e=a(),g=i("p"),g.textContent=Pt,Re=a(),M(C.$$.fragment),Qe=a(),M(B.$$.fragment),Ee=a(),M(v.$$.fragment),He=a(),M(G.$$.fragment),Ye=a(),x=i("p"),x.textContent=Dt,ze=a(),Z=i("p"),Z.textContent=Kt,Fe=a(),V=i("p"),V.textContent=Ot,Le=a(),X=i("p"),X.textContent=el,Ae=a(),W=i("p"),W.innerHTML=tl,Se=a(),_=i("p"),_.textContent=ll,Pe=a(),N=i("p"),N.innerHTML=sl,De=a(),M(q.$$.fragment),Ke=a(),k=i("p"),k.innerHTML=al,Oe=a(),M($.$$.fragment),et=a(),R=i("pre"),R.textContent=nl,tt=a(),Q=i("p"),Q.textContent=il,lt=a(),E=i("p"),E.textContent=ol,st=a(),H=i("p"),H.textContent=pl,at=a(),Y=i("ul"),Y.innerHTML=Ml,nt=a(),z=i("p"),z.innerHTML=rl,it=a(),M(F.$$.fragment),ot=a(),M(L.$$.fragment),pt=a(),A=i("p"),A.textContent=ul,Mt=a(),S=i("ul"),S.innerHTML=yl,rt=a(),M(P.$$.fragment),ut=a(),M(D.$$.fragment),yt=a(),M(K.$$.fragment),ct=a(),M(O.$$.fragment),mt=a(),ee=i("pre"),ee.textContent=cl,Jt=a(),te=i("p"),te.textContent=ml,dt=a(),le=i("p"),le.textContent=Jl,ht=a(),M(se.$$.fragment),wt=a(),ae=i("p"),ae.innerHTML=dl,Ut=a(),ne=i("ul"),ne.innerHTML=hl,jt=a(),M(ie.$$.fragment),Tt=a(),M(oe.$$.fragment),bt=a(),M(pe.$$.fragment),It=a(),Me=i("pre"),Me.textContent=wl,ft=a(),re=i("p"),re.innerHTML=Ul,gt=a(),ue=i("p"),ue.textContent=jl,Ct=a(),ye=i("p"),ye.textContent=Tl,Bt=a(),M(ce.$$.fragment),vt=a(),me=i("p"),me.textContent=bl,Gt=a(),M(Je.$$.fragment),xt=a(),de=i("p"),de.innerHTML=Il,Zt=a(),he=i("p"),he.innerHTML=fl,Vt=a(),we=i("p"),we.innerHTML=gl,Xt=a(),Ue=i("p"),Ue.innerHTML=Cl,Wt=a(),M(je.$$.fragment),_t=a(),Te=i("p"),Te.innerHTML=Bl,Nt=a(),be=i("p"),be.innerHTML=vl,qt=a(),M(Ie.$$.fragment),kt=a(),fe=i("p"),fe.textContent=Gl,$t=a(),ge=i("p"),ge.textContent=xl,Rt=a(),M(Ce.$$.fragment),Qt=a(),Ge=i("p"),this.h()},l(e){const t=kl("svelte-u9bgzb",document.head);d=o(t,"META",{name:!0,content:!0}),t.forEach(l),xe=n(e),ve=o(e,"P",{}),Zl(ve).forEach(l),Ze=n(e),r(h.$$.fragment,e),Ve=n(e),r(w.$$.fragment,e),Xe=n(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-1xlqnsv"&&(U.innerHTML=Yt),We=n(e),j=o(e,"P",{"data-svelte-h":!0}),p(j)!=="svelte-1c6d8zi"&&(j.textContent=zt),_e=n(e),T=o(e,"UL",{"data-svelte-h":!0}),p(T)!=="svelte-19yvpia"&&(T.innerHTML=Ft),Ne=n(e),b=o(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-qkxhb3"&&(b.textContent=Lt),qe=n(e),I=o(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-mjdvkr"&&(I.innerHTML=At),ke=n(e),f=o(e,"P",{"data-svelte-h":!0}),p(f)!=="svelte-1hxfecb"&&(f.textContent=St),$e=n(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-2ceznc"&&(g.textContent=Pt),Re=n(e),r(C.$$.fragment,e),Qe=n(e),r(B.$$.fragment,e),Ee=n(e),r(v.$$.fragment,e),He=n(e),r(G.$$.fragment,e),Ye=n(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-1pp2yf4"&&(x.textContent=Dt),ze=n(e),Z=o(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-m5cish"&&(Z.textContent=Kt),Fe=n(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-1s4xcl0"&&(V.textContent=Ot),Le=n(e),X=o(e,"P",{"data-svelte-h":!0}),p(X)!=="svelte-ocyvof"&&(X.textContent=el),Ae=n(e),W=o(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-ozh80s"&&(W.innerHTML=tl),Se=n(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-2p2co4"&&(_.textContent=ll),Pe=n(e),N=o(e,"P",{"data-svelte-h":!0}),p(N)!=="svelte-1e0wy16"&&(N.innerHTML=sl),De=n(e),r(q.$$.fragment,e),Ke=n(e),k=o(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-1jhfeb5"&&(k.innerHTML=al),Oe=n(e),r($.$$.fragment,e),et=n(e),R=o(e,"PRE",{"data-svelte-h":!0}),p(R)!=="svelte-es8io4"&&(R.textContent=nl),tt=n(e),Q=o(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-ls0opf"&&(Q.textContent=il),lt=n(e),E=o(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1d3x6kj"&&(E.textContent=ol),st=n(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-12akos5"&&(H.textContent=pl),at=n(e),Y=o(e,"UL",{"data-svelte-h":!0}),p(Y)!=="svelte-1esv9k9"&&(Y.innerHTML=Ml),nt=n(e),z=o(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1eueglv"&&(z.innerHTML=rl),it=n(e),r(F.$$.fragment,e),ot=n(e),r(L.$$.fragment,e),pt=n(e),A=o(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-1qmq503"&&(A.textContent=ul),Mt=n(e),S=o(e,"UL",{"data-svelte-h":!0}),p(S)!=="svelte-109y9bk"&&(S.innerHTML=yl),rt=n(e),r(P.$$.fragment,e),ut=n(e),r(D.$$.fragment,e),yt=n(e),r(K.$$.fragment,e),ct=n(e),r(O.$$.fragment,e),mt=n(e),ee=o(e,"PRE",{"data-svelte-h":!0}),p(ee)!=="svelte-6p10wk"&&(ee.textContent=cl),Jt=n(e),te=o(e,"P",{"data-svelte-h":!0}),p(te)!=="svelte-1l0eo7m"&&(te.textContent=ml),dt=n(e),le=o(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-zl50r1"&&(le.textContent=Jl),ht=n(e),r(se.$$.fragment,e),wt=n(e),ae=o(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-2kv0js"&&(ae.innerHTML=dl),Ut=n(e),ne=o(e,"UL",{"data-svelte-h":!0}),p(ne)!=="svelte-1byvfqn"&&(ne.innerHTML=hl),jt=n(e),r(ie.$$.fragment,e),Tt=n(e),r(oe.$$.fragment,e),bt=n(e),r(pe.$$.fragment,e),It=n(e),Me=o(e,"PRE",{"data-svelte-h":!0}),p(Me)!=="svelte-1v7ry63"&&(Me.textContent=wl),ft=n(e),re=o(e,"P",{"data-svelte-h":!0}),p(re)!=="svelte-1qt65cx"&&(re.innerHTML=Ul),gt=n(e),ue=o(e,"P",{"data-svelte-h":!0}),p(ue)!=="svelte-1uz9k2"&&(ue.textContent=jl),Ct=n(e),ye=o(e,"P",{"data-svelte-h":!0}),p(ye)!=="svelte-2se93w"&&(ye.textContent=Tl),Bt=n(e),r(ce.$$.fragment,e),vt=n(e),me=o(e,"P",{"data-svelte-h":!0}),p(me)!=="svelte-1qclzp5"&&(me.textContent=bl),Gt=n(e),r(Je.$$.fragment,e),xt=n(e),de=o(e,"P",{"data-svelte-h":!0}),p(de)!=="svelte-1o2xjbx"&&(de.innerHTML=Il),Zt=n(e),he=o(e,"P",{"data-svelte-h":!0}),p(he)!=="svelte-3qbx0g"&&(he.innerHTML=fl),Vt=n(e),we=o(e,"P",{"data-svelte-h":!0}),p(we)!=="svelte-ys5vz8"&&(we.innerHTML=gl),Xt=n(e),Ue=o(e,"P",{"data-svelte-h":!0}),p(Ue)!=="svelte-nftl58"&&(Ue.innerHTML=Cl),Wt=n(e),r(je.$$.fragment,e),_t=n(e),Te=o(e,"P",{"data-svelte-h":!0}),p(Te)!=="svelte-pp7myb"&&(Te.innerHTML=Bl),Nt=n(e),be=o(e,"P",{"data-svelte-h":!0}),p(be)!=="svelte-xhrhi7"&&(be.innerHTML=vl),qt=n(e),r(Ie.$$.fragment,e),kt=n(e),fe=o(e,"P",{"data-svelte-h":!0}),p(fe)!=="svelte-1bt3y9y"&&(fe.textContent=Gl),$t=n(e),ge=o(e,"P",{"data-svelte-h":!0}),p(ge)!=="svelte-7adf3t"&&(ge.textContent=xl),Rt=n(e),r(Ce.$$.fragment,e),Qt=n(e),Ge=o(e,"P",{}),Zl(Ge).forEach(l),this.h()},h(){Vl(d,"name","hf:doc:metadata"),Vl(d,"content",Hl)},m(e,t){$l(document.head,d),s(e,xe,t),s(e,ve,t),s(e,Ze,t),u(h,e,t),s(e,Ve,t),u(w,e,t),s(e,Xe,t),s(e,U,t),s(e,We,t),s(e,j,t),s(e,_e,t),s(e,T,t),s(e,Ne,t),s(e,b,t),s(e,qe,t),s(e,I,t),s(e,ke,t),s(e,f,t),s(e,$e,t),s(e,g,t),s(e,Re,t),u(C,e,t),s(e,Qe,t),u(B,e,t),s(e,Ee,t),u(v,e,t),s(e,He,t),u(G,e,t),s(e,Ye,t),s(e,x,t),s(e,ze,t),s(e,Z,t),s(e,Fe,t),s(e,V,t),s(e,Le,t),s(e,X,t),s(e,Ae,t),s(e,W,t),s(e,Se,t),s(e,_,t),s(e,Pe,t),s(e,N,t),s(e,De,t),u(q,e,t),s(e,Ke,t),s(e,k,t),s(e,Oe,t),u($,e,t),s(e,et,t),s(e,R,t),s(e,tt,t),s(e,Q,t),s(e,lt,t),s(e,E,t),s(e,st,t),s(e,H,t),s(e,at,t),s(e,Y,t),s(e,nt,t),s(e,z,t),s(e,it,t),u(F,e,t),s(e,ot,t),u(L,e,t),s(e,pt,t),s(e,A,t),s(e,Mt,t),s(e,S,t),s(e,rt,t),u(P,e,t),s(e,ut,t),u(D,e,t),s(e,yt,t),u(K,e,t),s(e,ct,t),u(O,e,t),s(e,mt,t),s(e,ee,t),s(e,Jt,t),s(e,te,t),s(e,dt,t),s(e,le,t),s(e,ht,t),u(se,e,t),s(e,wt,t),s(e,ae,t),s(e,Ut,t),s(e,ne,t),s(e,jt,t),u(ie,e,t),s(e,Tt,t),u(oe,e,t),s(e,bt,t),u(pe,e,t),s(e,It,t),s(e,Me,t),s(e,ft,t),s(e,re,t),s(e,gt,t),s(e,ue,t),s(e,Ct,t),s(e,ye,t),s(e,Bt,t),u(ce,e,t),s(e,vt,t),s(e,me,t),s(e,Gt,t),u(Je,e,t),s(e,xt,t),s(e,de,t),s(e,Zt,t),s(e,he,t),s(e,Vt,t),s(e,we,t),s(e,Xt,t),s(e,Ue,t),s(e,Wt,t),u(je,e,t),s(e,_t,t),s(e,Te,t),s(e,Nt,t),s(e,be,t),s(e,qt,t),u(Ie,e,t),s(e,kt,t),s(e,fe,t),s(e,$t,t),s(e,ge,t),s(e,Rt,t),u(Ce,e,t),s(e,Qt,t),s(e,Ge,t),Et=!0},p:Wl,i(e){Et||(y(h.$$.fragment,e),y(w.$$.fragment,e),y(C.$$.fragment,e),y(B.$$.fragment,e),y(v.$$.fragment,e),y(G.$$.fragment,e),y(q.$$.fragment,e),y($.$$.fragment,e),y(F.$$.fragment,e),y(L.$$.fragment,e),y(P.$$.fragment,e),y(D.$$.fragment,e),y(K.$$.fragment,e),y(O.$$.fragment,e),y(se.$$.fragment,e),y(ie.$$.fragment,e),y(oe.$$.fragment,e),y(pe.$$.fragment,e),y(ce.$$.fragment,e),y(Je.$$.fragment,e),y(je.$$.fragment,e),y(Ie.$$.fragment,e),y(Ce.$$.fragment,e),Et=!0)},o(e){c(h.$$.fragment,e),c(w.$$.fragment,e),c(C.$$.fragment,e),c(B.$$.fragment,e),c(v.$$.fragment,e),c(G.$$.fragment,e),c(q.$$.fragment,e),c($.$$.fragment,e),c(F.$$.fragment,e),c(L.$$.fragment,e),c(P.$$.fragment,e),c(D.$$.fragment,e),c(K.$$.fragment,e),c(O.$$.fragment,e),c(se.$$.fragment,e),c(ie.$$.fragment,e),c(oe.$$.fragment,e),c(pe.$$.fragment,e),c(ce.$$.fragment,e),c(Je.$$.fragment,e),c(je.$$.fragment,e),c(Ie.$$.fragment,e),c(Ce.$$.fragment,e),Et=!1},d(e){e&&(l(xe),l(ve),l(Ze),l(Ve),l(Xe),l(U),l(We),l(j),l(_e),l(T),l(Ne),l(b),l(qe),l(I),l(ke),l(f),l($e),l(g),l(Re),l(Qe),l(Ee),l(He),l(Ye),l(x),l(ze),l(Z),l(Fe),l(V),l(Le),l(X),l(Ae),l(W),l(Se),l(_),l(Pe),l(N),l(De),l(Ke),l(k),l(Oe),l(et),l(R),l(tt),l(Q),l(lt),l(E),l(st),l(H),l(at),l(Y),l(nt),l(z),l(it),l(ot),l(pt),l(A),l(Mt),l(S),l(rt),l(ut),l(yt),l(ct),l(mt),l(ee),l(Jt),l(te),l(dt),l(le),l(ht),l(wt),l(ae),l(Ut),l(ne),l(jt),l(Tt),l(bt),l(It),l(Me),l(ft),l(re),l(gt),l(ue),l(Ct),l(ye),l(Bt),l(vt),l(me),l(Gt),l(xt),l(de),l(Zt),l(he),l(Vt),l(we),l(Xt),l(Ue),l(Wt),l(_t),l(Te),l(Nt),l(be),l(qt),l(kt),l(fe),l($t),l(ge),l(Rt),l(Qt),l(Ge)),l(d),m(h,e),m(w,e),m(C,e),m(B,e),m(v,e),m(G,e),m(q,e),m($,e),m(F,e),m(L,e),m(P,e),m(D,e),m(K,e),m(O,e),m(se,e),m(ie,e),m(oe,e),m(pe,e),m(ce,e),m(Je,e),m(je,e),m(Ie,e),m(Ce,e)}}}const Hl='{"title":"Using LLM-as-a-judge 🧑‍⚖️ for an automated and versatile evaluation","local":"using-llm-as-a-judge--for-an-automated-and-versatile-evaluation","sections":[{"title":"1. Prepare the creation and evaluation of our LLM judge","local":"1-prepare-the-creation-and-evaluation-of-our-llm-judge","sections":[],"depth":2},{"title":"2. Create our LLM judge","local":"2-create-our-llm-judge","sections":[],"depth":2},{"title":"3. Improve the LLM judge","local":"3-improve-the-llm-judge","sections":[],"depth":2},{"title":"4. How do we take our LLM judge even further?","local":"4-how-do-we-take-our-llm-judge-even-further","sections":[],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2}],"depth":1}';function Yl(Ht){return _l(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Sl extends Nl{constructor(d){super(),ql(this,d,Yl,El,Xl,{})}}export{Sl as component};
