code: """
  Creates RAG dataset for tutorial notebooks and persists to disk.
  """
  
  import argparse
  import logging
  import sys
  from typing import List, Optional
  
  import llama_index
  import numpy as np
  import pandas as pd
  from gcsfs import GCSFileSystem
  from llama_index import ServiceContext, StorageContext, load_index_from_storage
  from llama_index.callbacks import CallbackManager, OpenInferenceCallbackHandler
  from llama_index.callbacks.open_inference_callback import as_dataframe
  from llama_index.embeddings.openai import OpenAIEmbedding
  from llama_index.llms import OpenAI
  from phoenix.experimental.evals.retrievals import (
      classify_relevance,
      compute_precisions_at_k,
  )
  from tqdm import tqdm
  
  
  def create_user_feedback(
      first_document_relevances: List[Optional[bool]],
      second_document_relevances: List[Optional[bool]],
  ) -> List[Optional[bool]]:
      """_summary_
  
      Args:
          first_document_relevances (List[Optional[bool]]): _description_
          second_document_relevances (List[Optional[bool]]): _description_
  
      Returns:
          List[Optional[bool]]: _description_
      """
      if len(first_document_relevances) != len(second_document_relevances):
          raise ValueError()
      first_document_relevances_array = np.array(first_document_relevances)
      second_document_relevances_array = np.array(second_document_relevances)
      failed_retrieval_mask = ~first_document_relevances_array & ~second_document_relevances_array
      num_failed_retrievals = failed_retrieval_mask.sum()
      num_thumbs_down = int(0.75 * num_failed_retrievals)
      failed_retrieval_indexes = np.where(failed_retrieval_mask)[0]
      thumbs_down_mask = np.random.choice(
          failed_retrieval_indexes, size=num_thumbs_down, replace=False
      )
      successful_retrieval_mask = ~failed_retrieval_mask
      num_successful_retrievals = successful_retrieval_mask.sum()
      num_thumbs_up = int(0.25 * num_successful_retrievals)
      successful_retrieval_indexes = np.where(successful_retrieval_mask)[0]
      thumbs_up_mask = np.random.choice(
          successful_retrieval_indexes, size=num_thumbs_up, replace=False
      )
      user_feedback_array = np.full(len(first_document_relevances), np.nan, dtype=np.float32)
      user_feedback_array[thumbs_down_mask] = -1.0
      user_feedback_array[thumbs_up_mask] = 1.0
      return [None if np.isnan(value) else value for value in user_feedback_array.tolist()]
  
  
  if __name__ == "__main__":
      logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
  
      parser = argparse.ArgumentParser()
      parser.add_argument("--index-path", type=str, required=True, help="Path to persisted index.")
      parser.add_argument(
          "--use-gcs",
          action="store_true",
          help="If this flag is set, the index will be loaded from GCS.",
      )
      parser.add_argument(
          "--query-path", type=str, required=True, help="Path to CSV file containing queries."
      )
      parser.add_argument(
          "--output-path", type=str, required=True, help="Path to output Parquet file."
      )
      args = parser.parse_args()
  
      llama_index.prompts.default_prompts.DEFAULT_TEXT_QA_PROMPT_TMPL = (
          "Context information is below.\n"
          "---------------------\n"
          "{context_str}\n"
          "---------------------\n"
          "Given the context information, "
          "answer the question and be as helpful as possible: {query_str}\n"
      )  # This prompt has been tweaked to make the system less conservative for demo purposes.
  
      queries = pd.read_csv(args.query_path)["Question"].tolist()
      file_system = GCSFileSystem(project="public-assets-275721") if args.use_gcs else None
      storage_context = StorageContext.from_defaults(
          fs=file_system,
          persist_dir=args.index_path,
      )
      callback_handler = OpenInferenceCallbackHandler()
      service_context = ServiceContext.from_defaults(
          llm=OpenAI(model="text-davinci-003"),
          embed_model=OpenAIEmbedding(model="text-embedding-ada-002"),
          callback_manager=CallbackManager(handlers=[callback_handler]),
      )
      index = load_index_from_storage(
          storage_context,
          service_context=service_context,
      )
      query_engine = index.as_query_engine()
  
      logging.info("Running queries")
      for query in tqdm(queries):
          query_engine.query(query)
  
      query_dataframe = as_dataframe(callback_handler.flush_query_data_buffer())
      document_dataframe = as_dataframe(callback_handler.flush_node_data_buffer())
  
      query_texts = query_dataframe[":feature.text:prompt"].tolist()
      list_of_document_id_lists = query_dataframe[
          ":feature.[str].retrieved_document_ids:prompt"
      ].tolist()
      document_id_to_text = dict(
          zip(document_dataframe["id"].to_list(), document_dataframe["node_text"].to_list())
      )
      first_document_texts, second_document_texts = [
          [
              document_id_to_text[document_ids[document_index]]
              for document_ids in list_of_document_id_lists
          ]
          for document_index in [0, 1]
      ]
  
      logging.info("Computing LLM-assisted ranking metrics")
      first_document_relevances, second_document_relevances = [
          [
              classify_relevance(query_text, document_text, model_name="gpt-4")
              for query_text, document_text in tqdm(zip(query_texts, first_document_texts))
          ]
          for document_texts in [first_document_texts, second_document_texts]
      ]
      list_of_precisions_at_k_lists = [
          compute_precisions_at_k([rel0, rel1])
          for rel0, rel1 in zip(first_document_relevances, second_document_relevances)
      ]
      precisions_at_1, precisions_at_2 = [
          [precisions_at_k[index] for precisions_at_k in list_of_precisions_at_k_lists]
          for index in [0, 1]
      ]
      document_similarity_0, document_similarity_1 = [
          [
              scores[index]
              for scores in query_dataframe[
                  ":feature.[float].retrieved_document_scores:prompt"
              ].tolist()
          ]
          for index in [0, 1]
      ]
      user_feedback = create_user_feedback(first_document_relevances, second_document_relevances)
      logging.info(
          f"Thumbs up: {sum([value == 1.0 for value in  user_feedback]) / len(user_feedback)}"
      )
      logging.info(
          f"Thumbs down: {sum([value == -1.0 for value in  user_feedback]) / len(user_feedback)}"
      )
  
      query_dataframe = query_dataframe.assign(
          **{
              ":tag.bool:relevance_0": first_document_relevances,
              ":tag.bool:relevance_1": second_document_relevances,
              ":tag.float:precision_at_1": precisions_at_1,
              ":tag.float:precision_at_2": precisions_at_2,
              ":tag.float:document_similarity_0": document_similarity_0,
              ":tag.float:document_similarity_1": document_similarity_1,
              ":tag.float:user_feedback": user_feedback,
          }
      )
      query_dataframe.to_parquet(args.output_path)
  
apis: llama_index.callbacks.OpenInferenceCallbackHandler llama_index.StorageContext.from_defaults llama_index.llms.OpenAI llama_index.load_index_from_storage llama_index.callbacks.CallbackManager llama_index.embeddings.openai.OpenAIEmbedding
extract_api: [((1235, 1270), 'numpy.array', 'np.array', (['first_document_relevances'], {}), '(first_document_relevances)\n', (1243, 1270), True, 'import numpy as np\n'), ((1310, 1346), 'numpy.array', 'np.array', (['second_document_relevances'], {}), '(second_document_relevances)\n', (1318, 1346), True, 'import numpy as np\n'), ((1645, 1724), 'numpy.random.choice', 'np.random.choice', (['failed_retrieval_indexes'], {'size': 'num_thumbs_down', 'replace': '(False)'}), '(failed_retrieval_indexes, size=num_thumbs_down, replace=False)\n', (1661, 1724), True, 'import numpy as np\n'), ((2011, 2097), 'numpy.random.choice', 'np.random.choice', (['successful_retrieval_indexes'], {'size': 'num_thumbs_up', 'replace': '(False)'}), '(successful_retrieval_indexes, size=num_thumbs_up, replace=\n    False)\n', (2027, 2097), True, 'import numpy as np\n'), ((2417, 2476), 'logging.basicConfig', 'logging.basicConfig', ([], {'level': 'logging.DEBUG', 'stream': 'sys.stdout'}), '(level=logging.DEBUG, stream=sys.stdout)\n', (2436, 2476), False, 'import logging\n'), ((2491, 2516), 'argparse.ArgumentParser', 'argparse.ArgumentParser', ([], {}), '()\n', (2514, 2516), False, 'import argparse\n'), ((3637, 3710), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'fs': 'file_system', 'persist_dir': 'args.index_path'}), '(fs=file_system, persist_dir=args.index_path)\n', (3665, 3710), False, 'from llama_index import ServiceContext, StorageContext, load_index_from_storage\n'), ((3757, 3787), 'llama_index.callbacks.OpenInferenceCallbackHandler', 'OpenInferenceCallbackHandler', ([], {}), '()\n', (3785, 3787), False, 'from llama_index.callbacks import CallbackManager, OpenInferenceCallbackHandler\n'), ((4044, 4117), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['storage_context'], {'service_context': 'service_context'}), '(storage_context, service_context=service_context)\n', (4067, 4117), False, 'from llama_index import ServiceContext, StorageContext, load_index_from_storage\n'), ((4189, 4220), 'logging.info', 'logging.info', (['"""Running queries"""'], {}), "('Running queries')\n", (4201, 4220), False, 'import logging\n'), ((4238, 4251), 'tqdm.tqdm', 'tqdm', (['queries'], {}), '(queries)\n', (4242, 4251), False, 'from tqdm import tqdm\n'), ((5004, 5058), 'logging.info', 'logging.info', (['"""Computing LLM-assisted ranking metrics"""'], {}), "('Computing LLM-assisted ranking metrics')\n", (5016, 5058), False, 'import logging\n'), ((1587, 1618), 'numpy.where', 'np.where', (['failed_retrieval_mask'], {}), '(failed_retrieval_mask)\n', (1595, 1618), True, 'import numpy as np\n'), ((1951, 1986), 'numpy.where', 'np.where', (['successful_retrieval_mask'], {}), '(successful_retrieval_mask)\n', (1959, 1986), True, 'import numpy as np\n'), ((3543, 3588), 'gcsfs.GCSFileSystem', 'GCSFileSystem', ([], {'project': '"""public-assets-275721"""'}), "(project='public-assets-275721')\n", (3556, 3588), False, 'from gcsfs import GCSFileSystem\n'), ((5437, 5474), 'phoenix.experimental.evals.retrievals.compute_precisions_at_k', 'compute_precisions_at_k', (['[rel0, rel1]'], {}), '([rel0, rel1])\n', (5460, 5474), False, 'from phoenix.experimental.evals.retrievals import classify_relevance, compute_precisions_at_k\n'), ((2314, 2329), 'numpy.isnan', 'np.isnan', (['value'], {}), '(value)\n', (2322, 2329), True, 'import numpy as np\n'), ((3852, 3884), 'llama_index.llms.OpenAI', 'OpenAI', ([], {'model': '"""text-davinci-003"""'}), "(model='text-davinci-003')\n", (3858, 3884), False, 'from llama_index.llms import OpenAI\n'), ((3906, 3953), 'llama_index.embeddings.openai.OpenAIEmbedding', 'OpenAIEmbedding', ([], {'model': '"""text-embedding-ada-002"""'}), "(model='text-embedding-ada-002')\n", (3921, 3953), False, 'from llama_index.embeddings.openai import OpenAIEmbedding\n'), ((3980, 4024), 'llama_index.callbacks.CallbackManager', 'CallbackManager', ([], {'handlers': '[callback_handler]'}), '(handlers=[callback_handler])\n', (3995, 4024), False, 'from llama_index.callbacks import CallbackManager, OpenInferenceCallbackHandler\n'), ((5143, 5208), 'phoenix.experimental.evals.retrievals.classify_relevance', 'classify_relevance', (['query_text', 'document_text'], {'model_name': '"""gpt-4"""'}), "(query_text, document_text, model_name='gpt-4')\n", (5161, 5208), False, 'from phoenix.experimental.evals.retrievals import classify_relevance, compute_precisions_at_k\n'), ((3475, 3503), 'pandas.read_csv', 'pd.read_csv', (['args.query_path'], {}), '(args.query_path)\n', (3486, 3503), True, 'import pandas as pd\n')]