from langchain.llms import OpenAI
from langchain import PromptTemplate, LLMChain
from getpass import getpass
OPENAI_API_KEY = getpass()
@serve.deployment
class DeployLLM:
    def __init__(self):
        # We initialize the LLM, template and the chain here
        llm = OpenAI(openai_api_key=OPENAI_API_KEY)
        template = "Question: {question}\n\nAnswer: Let's think step by step."
        prompt = PromptTemplate(template=template, input_variables=["question"])
        self.chain = LLMChain(llm=llm, prompt=prompt)
    def _run_chain(self, text: str):
        return self.chain(text)
    async def __call__(self, request: Request):
        # 1. Parse the request
        text = request.query_params["text"]
        # 2. Run the chain
        resp = self._run_chain(text)
        # 3. Return the response
        return resp["text"]
Now we can bind the deployment.
# Bind the model to deployment
deployment = DeployLLM.bind()
We can assign the port number and host when we want to run the deployment.
# Example port number
PORT_NUMBER = 8282
# Run the deployment
serve.api.run(deployment, port=PORT_NUMBER)
Now that service is deployed on port localhost:8282 we can send a post request to get the results back.
import requests
text = "What NFL team won the Super Bowl in the year Justin Beiber was born?"
response = requests.post(f'http://localhost:{PORT_NUMBER}/?text={text}')
print(response.content.decode())
previous
Qdrant
next
Rebuff
 Contents
  
Goal of this notebook
Setup Ray Serve
General Skeleton
rtdocs_stable/api.python.langchain.com/en/stable/integrations/ray_serve.html