See Generating Test Datasets at the end of this notebook for more details.
# import re
# from langchain.prompts import PromptTemplate
# template = """Below is a service description:
# {spec}
# Imagine you're a new user trying to use {operation} through a search bar. What are 10 different things you want to request?
# Wants/Questions:
# 1. """
# prompt = PromptTemplate.from_template(template)
# generation_chain = LLMChain(llm=llm, prompt=prompt)
# questions_ = generation_chain.run(spec=operation.to_typescript(), operation=operation.operation_id).split('\n')
# # Strip preceding numeric bullets
# questions = [re.sub(r'^\d+\. ', '', q).strip() for q in questions_]
# questions
# ground_truths = [
# {"q": ...} # What are the best queries for each input?
# ]
Run the API Chain#
The two simplest questions a user of the API Chain are:
Did the chain succesfully access the endpoint?
Did the action accomplish the correct result?
from collections import defaultdict
# Collect metrics to report at completion
scores = defaultdict(list)
from langchain.evaluation.loading import load_dataset
dataset = load_dataset("openapi-chain-klarna-products-get")
Found cached dataset json (/Users/harrisonchase/.cache/huggingface/datasets/LangChainDatasets___json/LangChainDatasets--openapi-chain-klarna-products-get-5d03362007667626/0.0.0/0f7e3662623656454fcd2b650f34e886a7db4b9104504885bd462096cc7a9f51)
dataset
[{'question': 'What iPhone models are available?',
rtdocs_stable/api.python.langchain.com/en/stable/use_cases/evaluation/openapi_eval.html