code: from PIL import Image
  import streamlit as st
  import openai
  #exercise 11
  from langchain.llms import OpenAI
  from langchain.prompts import PromptTemplate
  from langchain.chains import LLMChain
  #exercise 12
  from langchain.memory import ConversationBufferWindowMemory
  #exercise 13
  from langchain.document_loaders import TextLoader,PyPDFLoader
  from langchain.embeddings.openai import OpenAIEmbeddings
  from langchain.vectorstores import LanceDB
  import lancedb
  import os
  import tempfile
  #exercise 15
  import sqlite3
  import pandas as pd
  from datetime import datetime
  #exercise 16
  from langchain.agents import ConversationalChatAgent, AgentExecutor
  from langchain.callbacks import StreamlitCallbackHandler
  from langchain.chat_models import ChatOpenAI
  from langchain.memory import ConversationBufferMemory
  from langchain.memory.chat_message_histories import StreamlitChatMessageHistory
  from langchain.tools import DuckDuckGoSearchRun
  #Exercise 17
  from langchain.agents import tool
  import json
  
  # os.environ["OPENAI_API_KEY"] = st.secrets["openapi_key"]
  # openai.api_key = st.secrets["openapi_key"]
  
  #Global ex 13
  cwd = os.getcwd()
  WORKING_DIRECTORY = os.path.join(cwd, "database")
  
  if not os.path.exists(WORKING_DIRECTORY):
  	os.makedirs(WORKING_DIRECTORY)
  #ex15
  DB_NAME = os.path.join(WORKING_DIRECTORY, "default_db")
  
  def ex15_initialise():
  	# Create or check for the 'database' directory in the current working directory
  	# Set DB_NAME to be within the 'database' directory at the top of main.py
  	# Connect to the SQLite database
  	conn = sqlite3.connect(DB_NAME)
  	cursor = conn.cursor()
  
  	# Conversation data table
  	cursor.execute(
  		"""
  		CREATE TABLE IF NOT EXISTS data_table (
  			id INTEGER PRIMARY KEY,
  			date TEXT NOT NULL UNIQUE,
  			username TEXT NOT NULL,
  			chatbot_ans TEXT NOT NULL,
  			user_prompt TEXT NOT NULL,
  			tokens TEXT
  		)
  	"""
  	)
  	conn.commit()
  	conn.close()
  
  def ex15_collect(username, chatbot_response, prompt):
  	# collect data from bot
  	conn = sqlite3.connect(DB_NAME)
  	cursor = conn.cursor()
  	now = datetime.now()  # Using ISO format for date
  	tokens = len(chatbot_response) * 1.3
  	cursor.execute(
  		"""
  		INSERT INTO data_table (date, username,chatbot_ans, user_prompt, tokens)
  		VALUES (?, ?, ?, ?, ?)
  	""",
  		(now, username, chatbot_response, prompt, tokens),
  	)
  	conn.commit()
  	conn.close()
  
  # implementing data collection and displaying
  def ex15():
  	# initialise database first
  	ex15_initialise()
  	# collect some data
  	ex15_collect("yoda", "I am Yoda. The Force is strong with you", "Who are you?")
  	# display data
  	# Connect to the specified database
  	conn = sqlite3.connect(DB_NAME)
  	cursor = conn.cursor()
  
  	# Fetch all data from data_table
  	cursor.execute("SELECT * FROM data_table")
  	rows = cursor.fetchall()
  	column_names = [description[0] for description in cursor.description]
  	df = pd.DataFrame(rows, columns=column_names)
  	st.dataframe(df)
  	conn.close()
  
  # smart agents accessing the internet for free
  # https://github.com/langchain-ai/streamlit-agent/blob/main/streamlit_agent/search_and_chat.py
  def ex16_agent_bot():
  	st.title("🦜 LangChain: Chat with internet search")
  
  	msgs = StreamlitChatMessageHistory()
  	memory = ConversationBufferMemory(
  		chat_memory=msgs,
  		return_messages=True,
  		memory_key="chat_history",
  		output_key="output",
  	)
  	if len(msgs.messages) == 0 or st.sidebar.button("Reset chat history"):
  		msgs.clear()
  		msgs.add_ai_message("How can I help you?")
  		st.session_state.steps = {}
  
  	avatars = {"human": "user", "ai": "assistant"}
  	for idx, msg in enumerate(msgs.messages):
  		with st.chat_message(avatars[msg.type]):
  			# Render intermediate steps if any were saved
  			for step in st.session_state.steps.get(str(idx), []):
  				if step[0].tool == "_Exception":
  					continue
  				with st.status(
  					f"**{step[0].tool}**: {step[0].tool_input}", state="complete"
  				):
  					st.write(step[0].log)
  					st.write(step[1])
  			st.write(msg.content)
  
  	if prompt := st.chat_input(placeholder="Enter a query on the Internet"):
  		st.chat_message("user").write(prompt)
  
  		llm = ChatOpenAI(
  			model_name="gpt-3.5-turbo", openai_api_key=openai.api_key, streaming=True
  		)
  		tools = [DuckDuckGoSearchRun(name="Search")]
  		chat_agent = ConversationalChatAgent.from_llm_and_tools(llm=llm, tools=tools)
  		executor = AgentExecutor.from_agent_and_tools(
  			agent=chat_agent,
  			tools=tools,
  			memory=memory,
  			return_intermediate_steps=True,
  			handle_parsing_errors=True,
  		)
  		with st.chat_message("assistant"):
  			st_cb = StreamlitCallbackHandler(st.container(), expand_new_thoughts=False)
  			response = executor(prompt, callbacks=[st_cb])
  			st.write(response["output"])
  			st.session_state.steps[str(len(msgs.messages) - 1)] = response[
  				"intermediate_steps"
  			]
  
  def upload_file_streamlit():
  	def get_file_extension(file_name):
  		return os.path.splitext(file_name)[1]
  
  	st.subheader("Upload your docs")
  
  	# Streamlit file uploader to accept file input
  	uploaded_file = st.file_uploader("Choose a file", type=["docx", "txt", "pdf"])
  
  	if uploaded_file:
  		# Reading file content
  		file_content = uploaded_file.read()
  
  		# Determine the suffix based on uploaded file's name
  		file_suffix = get_file_extension(uploaded_file.name)
  
  		# Saving the uploaded file temporarily to process it
  		with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as temp_file:
  			temp_file.write(file_content)
  			temp_file.flush()  # Ensure the data is written to the file
  			temp_file_path = temp_file.name
  		return temp_file_path
  	
  #exercise 13 - split and chunk, embeddings and storing in vectorstores for reference
  def vectorstore_creator():
  	# WORKING_DIRECTORY set above in the main.py
  	# Process the temporary file using UnstructuredFileLoader (or any other method you need)
  	embeddings = OpenAIEmbeddings()
  	db = lancedb.connect(WORKING_DIRECTORY)
  	table = db.create_table(
  		"my_table",
  		data=[
  			{
  				"vector": embeddings.embed_query("Query unsuccessful"),
  				"text": "Query unsuccessful",
  				"id": "1",
  			}
  		],
  		mode="overwrite",
  	)
  	# st.write(temp_file_path)
  	temp_file_path = upload_file_streamlit()
  	if temp_file_path:
  		loader = PyPDFLoader(temp_file_path)
  		documents = loader.load_and_split()
  		db = LanceDB.from_documents(documents, embeddings, connection=table)
  		return db
  	
  # agents ,vectorstores, wiki
  # https://python.langchain.com/docs/modules/agents/how_to/custom_agent_with_tool_retrieval
  # note tool
  @tool("Document search")
  def document_search(query: str) -> str:
  	# this is the prompt to the tool itself
  	"Use this function first to search for documents pertaining to the query before going into the internet"
  	docs = st.session_state.vectorstore.similarity_search(query)
  	docs = docs[0].page_content
  	json_string = json.dumps(docs, ensure_ascii=False, indent=4)
  	return json_string
  
  # combine vector store and internet search
  def ex17_agent_bot():
  	st.title("🦜 LangChain: Chat with internet search")
  
  	st.session_state.vectorstore = vectorstore_creator()
  
  	msgs = StreamlitChatMessageHistory()
  	memory = ConversationBufferMemory(
  		chat_memory=msgs,
  		return_messages=True,
  		memory_key="chat_history",
  		output_key="output",
  	)
  	if len(msgs.messages) == 0 or st.sidebar.button("Reset chat history"):
  		msgs.clear()
  		msgs.add_ai_message("How can I help you?")
  		st.session_state.steps = {}
  
  	avatars = {"human": "user", "ai": "assistant"}
  	for idx, msg in enumerate(msgs.messages):
  		with st.chat_message(avatars[msg.type]):
  			# Render intermediate steps if any were saved
  			for step in st.session_state.steps.get(str(idx), []):
  				if step[0].tool == "_Exception":
  					continue
  				with st.status(
  					f"**{step[0].tool}**: {step[0].tool_input}", state="complete"
  				):
  					st.write(step[0].log)
  					st.write(step[1])
  			st.write(msg.content)
  
  	if prompt := st.chat_input(placeholder="Enter a query on the Internet"):
  		st.chat_message("user").write(prompt)
  
  		llm = ChatOpenAI(
  			model_name="gpt-3.5-turbo", openai_api_key=openai.api_key, streaming=True
  		)
  		tools = [document_search, DuckDuckGoSearchRun(name="Internet Search")]
  		chat_agent = ConversationalChatAgent.from_llm_and_tools(llm=llm, tools=tools)
  		executor = AgentExecutor.from_agent_and_tools(
  			agent=chat_agent,
  			tools=tools,
  			memory=memory,
  			return_intermediate_steps=True,
  			handle_parsing_errors=True,
  		)
  		with st.chat_message("assistant"):
  			st_cb = StreamlitCallbackHandler(st.container(), expand_new_thoughts=False)
  			response = executor(prompt, callbacks=[st_cb])
  			st.write(response["output"])
  			st.session_state.steps[str(len(msgs.messages) - 1)] = response[
  				"intermediate_steps"
  			]
  
apis: lancedb.connect
extract_api: [((1106, 1117), 'os.getcwd', 'os.getcwd', ([], {}), '()\n', (1115, 1117), False, 'import os\n'), ((1138, 1167), 'os.path.join', 'os.path.join', (['cwd', '"""database"""'], {}), "(cwd, 'database')\n", (1150, 1167), False, 'import os\n'), ((1259, 1304), 'os.path.join', 'os.path.join', (['WORKING_DIRECTORY', '"""default_db"""'], {}), "(WORKING_DIRECTORY, 'default_db')\n", (1271, 1304), False, 'import os\n'), ((6373, 6396), 'langchain.agents.tool', 'tool', (['"""Document search"""'], {}), "('Document search')\n", (6377, 6396), False, 'from langchain.agents import tool\n'), ((1176, 1209), 'os.path.exists', 'os.path.exists', (['WORKING_DIRECTORY'], {}), '(WORKING_DIRECTORY)\n', (1190, 1209), False, 'import os\n'), ((1212, 1242), 'os.makedirs', 'os.makedirs', (['WORKING_DIRECTORY'], {}), '(WORKING_DIRECTORY)\n', (1223, 1242), False, 'import os\n'), ((1527, 1551), 'sqlite3.connect', 'sqlite3.connect', (['DB_NAME'], {}), '(DB_NAME)\n', (1542, 1551), False, 'import sqlite3\n'), ((1957, 1981), 'sqlite3.connect', 'sqlite3.connect', (['DB_NAME'], {}), '(DB_NAME)\n', (1972, 1981), False, 'import sqlite3\n'), ((2013, 2027), 'datetime.datetime.now', 'datetime.now', ([], {}), '()\n', (2025, 2027), False, 'from datetime import datetime\n'), ((2579, 2603), 'sqlite3.connect', 'sqlite3.connect', (['DB_NAME'], {}), '(DB_NAME)\n', (2594, 2603), False, 'import sqlite3\n'), ((2810, 2850), 'pandas.DataFrame', 'pd.DataFrame', (['rows'], {'columns': 'column_names'}), '(rows, columns=column_names)\n', (2822, 2850), True, 'import pandas as pd\n'), ((2852, 2868), 'streamlit.dataframe', 'st.dataframe', (['df'], {}), '(df)\n', (2864, 2868), True, 'import streamlit as st\n'), ((3049, 3099), 'streamlit.title', 'st.title', (['"""🦜 LangChain: Chat with internet search"""'], {}), "('🦜 LangChain: Chat with internet search')\n", (3057, 3099), True, 'import streamlit as st\n'), ((3109, 3138), 'langchain.memory.chat_message_histories.StreamlitChatMessageHistory', 'StreamlitChatMessageHistory', ([], {}), '()\n', (3136, 3138), False, 'from langchain.memory.chat_message_histories import StreamlitChatMessageHistory\n'), ((3149, 3266), 'langchain.memory.ConversationBufferMemory', 'ConversationBufferMemory', ([], {'chat_memory': 'msgs', 'return_messages': '(True)', 'memory_key': '"""chat_history"""', 'output_key': '"""output"""'}), "(chat_memory=msgs, return_messages=True, memory_key\n    ='chat_history', output_key='output')\n", (3173, 3266), False, 'from langchain.memory import ConversationBufferMemory\n'), ((4818, 4850), 'streamlit.subheader', 'st.subheader', (['"""Upload your docs"""'], {}), "('Upload your docs')\n", (4830, 4850), True, 'import streamlit as st\n'), ((4917, 4979), 'streamlit.file_uploader', 'st.file_uploader', (['"""Choose a file"""'], {'type': "['docx', 'txt', 'pdf']"}), "('Choose a file', type=['docx', 'txt', 'pdf'])\n", (4933, 4979), True, 'import streamlit as st\n'), ((5732, 5750), 'langchain.embeddings.openai.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (5748, 5750), False, 'from langchain.embeddings.openai import OpenAIEmbeddings\n'), ((5757, 5791), 'lancedb.connect', 'lancedb.connect', (['WORKING_DIRECTORY'], {}), '(WORKING_DIRECTORY)\n', (5772, 5791), False, 'import lancedb\n'), ((6592, 6645), 'streamlit.session_state.vectorstore.similarity_search', 'st.session_state.vectorstore.similarity_search', (['query'], {}), '(query)\n', (6638, 6645), True, 'import streamlit as st\n'), ((6690, 6736), 'json.dumps', 'json.dumps', (['docs'], {'ensure_ascii': '(False)', 'indent': '(4)'}), '(docs, ensure_ascii=False, indent=4)\n', (6700, 6736), False, 'import json\n'), ((6824, 6874), 'streamlit.title', 'st.title', (['"""🦜 LangChain: Chat with internet search"""'], {}), "('🦜 LangChain: Chat with internet search')\n", (6832, 6874), True, 'import streamlit as st\n'), ((6939, 6968), 'langchain.memory.chat_message_histories.StreamlitChatMessageHistory', 'StreamlitChatMessageHistory', ([], {}), '()\n', (6966, 6968), False, 'from langchain.memory.chat_message_histories import StreamlitChatMessageHistory\n'), ((6979, 7096), 'langchain.memory.ConversationBufferMemory', 'ConversationBufferMemory', ([], {'chat_memory': 'msgs', 'return_messages': '(True)', 'memory_key': '"""chat_history"""', 'output_key': '"""output"""'}), "(chat_memory=msgs, return_messages=True, memory_key\n    ='chat_history', output_key='output')\n", (7003, 7096), False, 'from langchain.memory import ConversationBufferMemory\n'), ((3305, 3344), 'streamlit.sidebar.button', 'st.sidebar.button', (['"""Reset chat history"""'], {}), "('Reset chat history')\n", (3322, 3344), True, 'import streamlit as st\n'), ((3912, 3970), 'streamlit.chat_input', 'st.chat_input', ([], {'placeholder': '"""Enter a query on the Internet"""'}), "(placeholder='Enter a query on the Internet')\n", (3925, 3970), True, 'import streamlit as st\n'), ((4021, 4110), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'model_name': '"""gpt-3.5-turbo"""', 'openai_api_key': 'openai.api_key', 'streaming': '(True)'}), "(model_name='gpt-3.5-turbo', openai_api_key=openai.api_key,\n    streaming=True)\n", (4031, 4110), False, 'from langchain.chat_models import ChatOpenAI\n'), ((4176, 4240), 'langchain.agents.ConversationalChatAgent.from_llm_and_tools', 'ConversationalChatAgent.from_llm_and_tools', ([], {'llm': 'llm', 'tools': 'tools'}), '(llm=llm, tools=tools)\n', (4218, 4240), False, 'from langchain.agents import ConversationalChatAgent, AgentExecutor\n'), ((4254, 4399), 'langchain.agents.AgentExecutor.from_agent_and_tools', 'AgentExecutor.from_agent_and_tools', ([], {'agent': 'chat_agent', 'tools': 'tools', 'memory': 'memory', 'return_intermediate_steps': '(True)', 'handle_parsing_errors': '(True)'}), '(agent=chat_agent, tools=tools, memory=\n    memory, return_intermediate_steps=True, handle_parsing_errors=True)\n', (4288, 4399), False, 'from langchain.agents import ConversationalChatAgent, AgentExecutor\n'), ((6089, 6116), 'langchain.document_loaders.PyPDFLoader', 'PyPDFLoader', (['temp_file_path'], {}), '(temp_file_path)\n', (6100, 6116), False, 'from langchain.document_loaders import TextLoader, PyPDFLoader\n'), ((6162, 6225), 'langchain.vectorstores.LanceDB.from_documents', 'LanceDB.from_documents', (['documents', 'embeddings'], {'connection': 'table'}), '(documents, embeddings, connection=table)\n', (6184, 6225), False, 'from langchain.vectorstores import LanceDB\n'), ((7135, 7174), 'streamlit.sidebar.button', 'st.sidebar.button', (['"""Reset chat history"""'], {}), "('Reset chat history')\n", (7152, 7174), True, 'import streamlit as st\n'), ((7742, 7800), 'streamlit.chat_input', 'st.chat_input', ([], {'placeholder': '"""Enter a query on the Internet"""'}), "(placeholder='Enter a query on the Internet')\n", (7755, 7800), True, 'import streamlit as st\n'), ((7851, 7940), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'model_name': '"""gpt-3.5-turbo"""', 'openai_api_key': 'openai.api_key', 'streaming': '(True)'}), "(model_name='gpt-3.5-turbo', openai_api_key=openai.api_key,\n    streaming=True)\n", (7861, 7940), False, 'from langchain.chat_models import ChatOpenAI\n'), ((8032, 8096), 'langchain.agents.ConversationalChatAgent.from_llm_and_tools', 'ConversationalChatAgent.from_llm_and_tools', ([], {'llm': 'llm', 'tools': 'tools'}), '(llm=llm, tools=tools)\n', (8074, 8096), False, 'from langchain.agents import ConversationalChatAgent, AgentExecutor\n'), ((8110, 8255), 'langchain.agents.AgentExecutor.from_agent_and_tools', 'AgentExecutor.from_agent_and_tools', ([], {'agent': 'chat_agent', 'tools': 'tools', 'memory': 'memory', 'return_intermediate_steps': '(True)', 'handle_parsing_errors': '(True)'}), '(agent=chat_agent, tools=tools, memory=\n    memory, return_intermediate_steps=True, handle_parsing_errors=True)\n', (8144, 8255), False, 'from langchain.agents import ConversationalChatAgent, AgentExecutor\n'), ((3535, 3569), 'streamlit.chat_message', 'st.chat_message', (['avatars[msg.type]'], {}), '(avatars[msg.type])\n', (3550, 3569), True, 'import streamlit as st\n'), ((3875, 3896), 'streamlit.write', 'st.write', (['msg.content'], {}), '(msg.content)\n', (3883, 3896), True, 'import streamlit as st\n'), ((4125, 4159), 'langchain.tools.DuckDuckGoSearchRun', 'DuckDuckGoSearchRun', ([], {'name': '"""Search"""'}), "(name='Search')\n", (4144, 4159), False, 'from langchain.tools import DuckDuckGoSearchRun\n'), ((4422, 4450), 'streamlit.chat_message', 'st.chat_message', (['"""assistant"""'], {}), "('assistant')\n", (4437, 4450), True, 'import streamlit as st\n'), ((4584, 4612), 'streamlit.write', 'st.write', (["response['output']"], {}), "(response['output'])\n", (4592, 4612), True, 'import streamlit as st\n'), ((4785, 4812), 'os.path.splitext', 'os.path.splitext', (['file_name'], {}), '(file_name)\n', (4801, 4812), False, 'import os\n'), ((5237, 5298), 'tempfile.NamedTemporaryFile', 'tempfile.NamedTemporaryFile', ([], {'delete': '(False)', 'suffix': 'file_suffix'}), '(delete=False, suffix=file_suffix)\n', (5264, 5298), False, 'import tempfile\n'), ((7365, 7399), 'streamlit.chat_message', 'st.chat_message', (['avatars[msg.type]'], {}), '(avatars[msg.type])\n', (7380, 7399), True, 'import streamlit as st\n'), ((7705, 7726), 'streamlit.write', 'st.write', (['msg.content'], {}), '(msg.content)\n', (7713, 7726), True, 'import streamlit as st\n'), ((7972, 8015), 'langchain.tools.DuckDuckGoSearchRun', 'DuckDuckGoSearchRun', ([], {'name': '"""Internet Search"""'}), "(name='Internet Search')\n", (7991, 8015), False, 'from langchain.tools import DuckDuckGoSearchRun\n'), ((8278, 8306), 'streamlit.chat_message', 'st.chat_message', (['"""assistant"""'], {}), "('assistant')\n", (8293, 8306), True, 'import streamlit as st\n'), ((8440, 8468), 'streamlit.write', 'st.write', (["response['output']"], {}), "(response['output'])\n", (8448, 8468), True, 'import streamlit as st\n'), ((3974, 3997), 'streamlit.chat_message', 'st.chat_message', (['"""user"""'], {}), "('user')\n", (3989, 3997), True, 'import streamlit as st\n'), ((4488, 4502), 'streamlit.container', 'st.container', ([], {}), '()\n', (4500, 4502), True, 'import streamlit as st\n'), ((7804, 7827), 'streamlit.chat_message', 'st.chat_message', (['"""user"""'], {}), "('user')\n", (7819, 7827), True, 'import streamlit as st\n'), ((8344, 8358), 'streamlit.container', 'st.container', ([], {}), '()\n', (8356, 8358), True, 'import streamlit as st\n'), ((3737, 3809), 'streamlit.status', 'st.status', (['f"""**{step[0].tool}**: {step[0].tool_input}"""'], {'state': '"""complete"""'}), "(f'**{step[0].tool}**: {step[0].tool_input}', state='complete')\n", (3746, 3809), True, 'import streamlit as st\n'), ((3827, 3848), 'streamlit.write', 'st.write', (['step[0].log'], {}), '(step[0].log)\n', (3835, 3848), True, 'import streamlit as st\n'), ((3854, 3871), 'streamlit.write', 'st.write', (['step[1]'], {}), '(step[1])\n', (3862, 3871), True, 'import streamlit as st\n'), ((7567, 7639), 'streamlit.status', 'st.status', (['f"""**{step[0].tool}**: {step[0].tool_input}"""'], {'state': '"""complete"""'}), "(f'**{step[0].tool}**: {step[0].tool_input}', state='complete')\n", (7576, 7639), True, 'import streamlit as st\n'), ((7657, 7678), 'streamlit.write', 'st.write', (['step[0].log'], {}), '(step[0].log)\n', (7665, 7678), True, 'import streamlit as st\n'), ((7684, 7701), 'streamlit.write', 'st.write', (['step[1]'], {}), '(step[1])\n', (7692, 7701), True, 'import streamlit as st\n')]