code: import matplotlib.pyplot as plt
  import polars as pl
  import seaborn as sns
  import torch
  from llama_index.evaluation import RelevancyEvaluator
  from llama_index.llms import HuggingFaceLLM
  from llama_index.prompts import PromptTemplate
  from tqdm import tqdm
  from transformers import BitsAndBytesConfig
  
  from src.common.utils import Settings
  from src.model import LlamaIndexModel
  
  pl.Config.set_tbl_formatting("NOTHING")
  pl.Config.set_tbl_rows(4)
  
  settings = Settings().model.model_dump()
  settings["top_k"] = 5  # reduce eval time
  
  model = LlamaIndexModel(**settings, load_model=True)
  
  
  quantization_config = BitsAndBytesConfig(
      load_in_4bit=True,
      bnb_4bit_compute_dtype=torch.float16,
      bnb_4bit_quant_type="nf4",
      bnb_4bit_use_double_quant=True,
  )
  model.model = HuggingFaceLLM(
      model_name="mistralai/Mistral-7B-Instruct-v0.1",
      tokenizer_name="mistralai/Mistral-7B-Instruct-v0.1",
      query_wrapper_prompt=PromptTemplate("<s>[INST] {query_str} [/INST] </s>\n"),
      context_window=3900,
      max_new_tokens=256,
      model_kwargs={"quantization_config": quantization_config},
      generate_kwargs={"temperature": 0.2, "top_k": 5, "top_p": 0.95},
      device_map="auto",
  )
  model.build_index()
  
  
  past_queries = (
      pl.read_csv("data/logs/queries.csv").filter(pl.col("column") != "").head(100)
  )
  
  fails = ["supercars"]  # these cases should always output 'false'
  queries = [
      "social mobility",
      "mobility",
      "diabetes",
      "health",
      "liverpool",
      "london",
      "covid",
      "greenspace",
  ] + fails
  queries.extend([f"{query} datasets" for query in queries])
  queries.extend([f"datasets relating to {query}" for query in queries])
  queries.extend(past_queries["column"].to_list())
  alpha_values = [0.0, 0.75, 1.0]
  
  results = []
  for alpha in tqdm(alpha_values):
      for query in tqdm(queries):
          query
          model.alpha = alpha
          model.run(query)
          evaluator = RelevancyEvaluator(service_context=model.service_context)
          contexts = [node.get_content() for node in model.response]
          eval_result = evaluator.evaluate(
              query=query,
              contexts=contexts,
              response="",
          )
          results.append({"result": eval_result.passing, "alpha": alpha, "query": query})
  
  df = pl.DataFrame(results).with_columns(
      pl.col("alpha").cast(str), pl.col("result").cast(str)
  )
  df.write_csv("data/evaluation/evaluation.csv")
  df = pl.read_csv("data/evaluation/evaluation.csv").with_columns(
      pl.col("alpha").cast(str), pl.col("result").cast(str)
  )
  
  sns.histplot(
      data=df,
      x="alpha",
      hue="result",
      multiple="stack",
      shrink=0.8,
      palette="gray",
  )
  plt.save("./data/evaluation/plot.png")
  
apis: llama_index.evaluation.RelevancyEvaluator llama_index.prompts.PromptTemplate
extract_api: [((376, 415), 'polars.Config.set_tbl_formatting', 'pl.Config.set_tbl_formatting', (['"""NOTHING"""'], {}), "('NOTHING')\n", (404, 415), True, 'import polars as pl\n'), ((416, 441), 'polars.Config.set_tbl_rows', 'pl.Config.set_tbl_rows', (['(4)'], {}), '(4)\n', (438, 441), True, 'import polars as pl\n'), ((535, 579), 'src.model.LlamaIndexModel', 'LlamaIndexModel', ([], {'load_model': '(True)'}), '(**settings, load_model=True)\n', (550, 579), False, 'from src.model import LlamaIndexModel\n'), ((604, 742), 'transformers.BitsAndBytesConfig', 'BitsAndBytesConfig', ([], {'load_in_4bit': '(True)', 'bnb_4bit_compute_dtype': 'torch.float16', 'bnb_4bit_quant_type': '"""nf4"""', 'bnb_4bit_use_double_quant': '(True)'}), "(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16,\n    bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True)\n", (622, 742), False, 'from transformers import BitsAndBytesConfig\n'), ((1766, 1784), 'tqdm.tqdm', 'tqdm', (['alpha_values'], {}), '(alpha_values)\n', (1770, 1784), False, 'from tqdm import tqdm\n'), ((2526, 2622), 'seaborn.histplot', 'sns.histplot', ([], {'data': 'df', 'x': '"""alpha"""', 'hue': '"""result"""', 'multiple': '"""stack"""', 'shrink': '(0.8)', 'palette': '"""gray"""'}), "(data=df, x='alpha', hue='result', multiple='stack', shrink=0.8,\n    palette='gray')\n", (2538, 2622), True, 'import seaborn as sns\n'), ((2646, 2684), 'matplotlib.pyplot.save', 'plt.save', (['"""./data/evaluation/plot.png"""'], {}), "('./data/evaluation/plot.png')\n", (2654, 2684), True, 'import matplotlib.pyplot as plt\n'), ((1803, 1816), 'tqdm.tqdm', 'tqdm', (['queries'], {}), '(queries)\n', (1807, 1816), False, 'from tqdm import tqdm\n'), ((923, 977), 'llama_index.prompts.PromptTemplate', 'PromptTemplate', (['"""<s>[INST] {query_str} [/INST] </s>\n"""'], {}), "('<s>[INST] {query_str} [/INST] </s>\\n')\n", (937, 977), False, 'from llama_index.prompts import PromptTemplate\n'), ((1905, 1962), 'llama_index.evaluation.RelevancyEvaluator', 'RelevancyEvaluator', ([], {'service_context': 'model.service_context'}), '(service_context=model.service_context)\n', (1923, 1962), False, 'from llama_index.evaluation import RelevancyEvaluator\n'), ((2257, 2278), 'polars.DataFrame', 'pl.DataFrame', (['results'], {}), '(results)\n', (2269, 2278), True, 'import polars as pl\n'), ((2405, 2450), 'polars.read_csv', 'pl.read_csv', (['"""data/evaluation/evaluation.csv"""'], {}), "('data/evaluation/evaluation.csv')\n", (2416, 2450), True, 'import polars as pl\n'), ((454, 464), 'src.common.utils.Settings', 'Settings', ([], {}), '()\n', (462, 464), False, 'from src.common.utils import Settings\n'), ((2297, 2312), 'polars.col', 'pl.col', (['"""alpha"""'], {}), "('alpha')\n", (2303, 2312), True, 'import polars as pl\n'), ((2324, 2340), 'polars.col', 'pl.col', (['"""result"""'], {}), "('result')\n", (2330, 2340), True, 'import polars as pl\n'), ((2469, 2484), 'polars.col', 'pl.col', (['"""alpha"""'], {}), "('alpha')\n", (2475, 2484), True, 'import polars as pl\n'), ((2496, 2512), 'polars.col', 'pl.col', (['"""result"""'], {}), "('result')\n", (2502, 2512), True, 'import polars as pl\n'), ((1228, 1264), 'polars.read_csv', 'pl.read_csv', (['"""data/logs/queries.csv"""'], {}), "('data/logs/queries.csv')\n", (1239, 1264), True, 'import polars as pl\n'), ((1272, 1288), 'polars.col', 'pl.col', (['"""column"""'], {}), "('column')\n", (1278, 1288), True, 'import polars as pl\n')]