code: import json
  import os
  import time
  import fitz  # PyMuPDF
  import llama_index
  import openai
  import weaviate
  from weaviate.gql.get import HybridFusion
  from unstructured.cleaners.core import clean
  from llama_index.vector_stores import WeaviateVectorStore
  from llama_index import VectorStoreIndex, ServiceContext, set_global_service_context
  from llama_index.response.pprint_utils import pprint_source_node
  import os
  
  import weaviate
  from langchain.document_loaders import GutenbergLoader
  from langchain.embeddings.openai import OpenAIEmbeddings
  from langchain.text_splitter import CharacterTextSplitter
  from langchain.vectorstores import Weaviate
  
  # Grimms' Fairy Tales by Jacob Grimm and Wilhelm Grimm
  loader = GutenbergLoader("https://www.gutenberg.org/files/2591/2591-0.txt")
  documents = loader.load()
  
  text_splitter = CharacterTextSplitter(
      chunk_size=500, chunk_overlap=0, length_function=len
  )
  docs = text_splitter.split_documents(documents)
  WEAVIATE_URL = "http://weaviate:8080"
  client = weaviate.Client(
      url=WEAVIATE_URL,
      additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
  )
  client.schema.delete_all()
  client.schema.get()
  schema = {
      "classes": [
          {
              "class": "Test",
              "description": "A written paragraph",
              "vectorizer": "text2vec-openai",
              "moduleConfig": {"text2vec-openai": {"model": "ada", "type": "text"}},
          },
      ]
  }
  client.schema.create(schema)
  vectorstore = Weaviate(client, "Paragraph", "content")
  
  text_meta_pair = [(doc.page_content, doc.metadata) for doc in docs]
  
  texts, meta = list(zip(*text_meta_pair))
  vectorstore.add_texts(texts, meta)
  query = "the part where with talking animals"
  docs = vectorstore.similarity_search(query)
  
  for doc in docs:
      print(doc.page_content)
      print("*" * 80)
  
  azure_openai_key = os.getenv("AZURE_OPENAI_KEY")
  azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
  resource_name = os.getenv("RESOURCE_NAME")
  azure_client = openai.lib.azure.AzureOpenAI(
      azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
      api_key=os.getenv("AZURE_OPENAI_KEY"),
      api_version="2023-05-15"
  )
  headers = {
      "X-Azure-Api-Key": azure_openai_key,
  }
  
  
  def query_openai(messages):
      return azure_client.chat.completions.create(
          model="gpt-35-16k",  # model = "deployment_name".
          messages=messages
      )
  
  
  def prompt(query):
      return f""" You are a university professor.
      Answer the following question using only the provided context.
      If you can't find the answer, do not pretend you know it, ask for more information"
      Answer in the same langauge as the question. If you used your own knowledge apart from the context provided mention that.
      Question:  {query} """
  
  
  def chunk_files(subdirectory_path, subdirectory):
      data = []
      # Process each PDF file in this subdirectory
      for filename in os.listdir(subdirectory_path):
          if filename.endswith('.pdf'):
              file_path = os.path.join(subdirectory_path, filename)
              str_five = ""
              # Open the PDF
              with fitz.open(file_path) as doc:
                  for page_num in range(len(doc)):
                      page_text = doc[page_num].get_text()
                      page_text = clean(page_text, bullets=True, extra_whitespace=True)
                      slide_id = filename + str(page_num)
                      if page_num % 5 == 0:
                          if page_num != 0:  # Avoid appending empty content for the first page
                              data.append({
                                  "content": str_five,
                                  "slide_id": slide_id,
                                  "page_interval": str(str(page_num - 5) + "->" + str(page_num)),
                                  "lecture_id": subdirectory  # Save the subdirectory name
                              })
                          last_page = doc[page_num - 1].get_text() if page_num > 0 else ""
                          last_page = clean(last_page, bullets=True, extra_whitespace=True)
                          str_five = last_page + page_text
                      else:
                          str_five += "\n\n" + page_text
                  # Append the last accumulated text if it's not empty
                  if str_five:
                      data.append({
                          "content": str_five,
                          "slide_id": subdirectory_path + str(len(doc)),
                          "page_interval": str(str(len(doc) - 10) + "->" + str(len(doc))),
                          "lecture_id": subdirectory  # Save the subdirectory name
                      })
      return data
  
  
  class AI:
      def __init__(self):
          api_key_header = {
              "X-Azure-Api-Key": azure_openai_key,  # Replace with your inference API key
          }
          self.client = weaviate.Client(
              url="http://localhost:8080",  # Replace with your endpoint
              additional_headers=api_key_header
          )
  
      def create_class(self):
          t2v = {
              "model": "ada",
              "modelVersion": "002",
              "type": "text",
              "baseURL": azure_endpoint,
              "resourceName": resource_name,
              "deploymentId": "te-ada-002",
          }
          self.client.schema.delete_class("Lectures")
          if not self.client.schema.exists("Lectures"):
              class_obj = {
                  "class": "Lectures",
                  "vectorizer": "text2vec-openai",
                  "properties": [
                      {
                          "dataType": ["text"],
                          "name": "content",
                          "moduleConfig": {
                              "text2vec-openai": {
                                  "vectorizePropertyName": False
                              }
                          },
                      },
                      {
                          "dataType": ["text"],
                          "name": "slide_id",
                          "moduleConfig": {
                              "text2vec-openai": {
                                  "vectorizePropertyName": False
                              }
                          },
                      },
                      {
                          "dataType": ["text"],
                          "name": "page_interval",
                          "moduleConfig": {
                              "text2vec-openai": {
                                  "vectorizePropertyName": False
                              }
                          },
                      },
                      {
                          "dataType": ["text"],
                          "name": "lecture_id",
                          "moduleConfig": {
                              "text2vec-openai": {
                                  "vectorizePropertyName": False
                              }
                          },
                      }
                  ],
                  "moduleConfig": {
                      "text2vec-openai": t2v,
                      "generative-openai": {
                          "baseURL": azure_endpoint,
                          "resourceName": resource_name,
                          "deploymentId": "gpt-35-16k",
                          "waitForModel": True,
                          "useGPU": False,
                          "useCache": True
                      }
                  },
              }
              self.client.schema.create_class(class_obj)
              print("Schema created")
              directory_path = "../../lectures"
              print("Importing data into the batch")
              # Iterate through each subdirectory in the root directory
              for subdirectory in os.listdir(directory_path):
                  subdirectory_path = os.path.join(directory_path, subdirectory)
                  if os.path.isdir(subdirectory_path):
                      self.batch_import(subdirectory_path, subdirectory)
              print("Import Finished")
  
      def batch_import(self, directory_path, subdirectory):
          data = chunk_files(directory_path, subdirectory)
          # Configure a batch process
          self.client.batch.configure(
              # `batch_size` takes an `int` value to enable auto-batching
              # dynamically update the `batch_size` based on import speed
              dynamic=True,
              timeout_retries=0
          )
          with self.client.batch as batch:
              # Batch import all Questions
  
              for i, d in enumerate(data):
                  embeddings_created = False
                  properties = {
                      "content": d["content"],
                      "slide_id": d["slide_id"],
                      "page_interval": d["page_interval"],
                      "lecture_id": d["lecture_id"]
                  }
                  # Initialize the flag
                  embeddings_created = False
                  # create embeddings (exponential backoff to avoid RateLimitError)
                  for j in range(5):  # max 5 retries
                      # Only attempt to create embeddings if not already created
                      if not embeddings_created:
                          try:
                              batch.add_data_object(
                                  properties,
                                  "Lectures"
                              )
                              embeddings_created = True  # Set flag to True on success
                              break  # Break the loop as embedding creation was successful
                          except openai.error.RateLimitError:
                              time.sleep(2 ** j)  # wait 2^j seconds before retrying
                              print("Retrying import...")
                      else:
                          break  # Exit loop if embeddings already created
  
                  # Raise an error if embeddings were not created after retries
                  if not embeddings_created:
                      raise RuntimeError("Failed to create embeddings.")
  
      def generate_response(self, user_message, lecture_id):
          completion = query_openai(messages=[{
              "role": "user",
              "content": f"""
                                                 Please give back lecture content that can answer this inquiry: 
                                                 Do not add anything else.
                                                 "{user_message}".\n
                                                 """}])
          generated_lecture = completion.choices[0].message.content
  
          if lecture_id == "CIT5230000":
              llm = llama_index.llms.AzureOpenAI(model="gpt-35-turbo-16k", deployment_name="gpt-35-16k",
                                                 api_key=azure_openai_key, azure_endpoint=azure_endpoint,
                                                 api_version="2023-03-15-preview")
              embed_model = llama_index.embeddings.AzureOpenAIEmbedding(
                  model="text-embedding-ada-002",
                  deployment_name="te-ada-002",
                  api_key=azure_openai_key,
                  azure_endpoint=azure_endpoint,
                  api_version="2023-03-15-preview"
              )
              service_context = ServiceContext.from_defaults(llm=llm,  embed_model=embed_model)
  
              vector_store = WeaviateVectorStore(
                  weaviate_client=self.client, index_name="Lectures", text_key="content"
              )
              retriever = VectorStoreIndex.from_vector_store(vector_store, service_context=service_context).as_retriever(
                  similarity_top_k=1
              )
              nodes = retriever.retrieve(generated_lecture)
              pprint_source_node(nodes[0])
              print(nodes[0].node.metadata)
  
          # add hypothetical document embeddings (hyde)
          if lecture_id != "" and lecture_id is not None:
              response = (
                  self.client.query
                  .get("Lectures", ["content", "slide_id", "page_interval", ])
                  .with_where({
                      "path": ["lecture_id"],
                      "operator": "Equal",
                      "valueText": lecture_id
                  })
                  .with_near_text({"concepts": generated_lecture})
                  #w.with_additional(f'rerank( query: "{user_message}", property: "content"){{score}}')
                  .with_generate(grouped_task=prompt(user_message))
                  .with_limit(1)
                  .do()
              )
              generated_response = response["data"]["Get"]["Lectures"][0]["_additional"]["generate"]["groupedResult"]
  
          else:
              response = (
                  self.client.query
                  .get("Lectures", ["content", "slide_id", "page_interval", "lecture_id"])
                  # alpha = 0 forces using a pure keyword search method (BM25)
                  # alpha = 1 forces using a pure vector search method
                  .with_hybrid(query=user_message,
                               alpha=1,
                               fusion_type=HybridFusion.RELATIVE_SCORE
  
                               )
                  #                .with_additional(f'rerank( query: "{user_message}", property: "content"){{score}}')
                  .with_generate(grouped_task=prompt(user_message))
                  .with_limit(3)
                  .do()
              )
              generated_response = response["data"]["Get"]["Lectures"][0]["_additional"]["generate"]["groupedResult"]
          slides = response["data"]["Get"]["Lectures"][0]["slide_id"]
          page_interval = response["data"]["Get"]["Lectures"][0]["page_interval"]
          print(json.dumps(response, indent=2))
          return generated_response + f"""\n\nMore relevant information on the  slides {slides} "pages":{page_interval} """
  
apis: llama_index.llms.AzureOpenAI llama_index.embeddings.AzureOpenAIEmbedding llama_index.ServiceContext.from_defaults llama_index.VectorStoreIndex.from_vector_store llama_index.response.pprint_utils.pprint_source_node llama_index.vector_stores.WeaviateVectorStore
extract_api: [((707, 773), 'langchain.document_loaders.GutenbergLoader', 'GutenbergLoader', (['"""https://www.gutenberg.org/files/2591/2591-0.txt"""'], {}), "('https://www.gutenberg.org/files/2591/2591-0.txt')\n", (722, 773), False, 'from langchain.document_loaders import GutenbergLoader\n'), ((817, 892), 'langchain.text_splitter.CharacterTextSplitter', 'CharacterTextSplitter', ([], {'chunk_size': '(500)', 'chunk_overlap': '(0)', 'length_function': 'len'}), '(chunk_size=500, chunk_overlap=0, length_function=len)\n', (838, 892), False, 'from langchain.text_splitter import CharacterTextSplitter\n'), ((994, 1102), 'weaviate.Client', 'weaviate.Client', ([], {'url': 'WEAVIATE_URL', 'additional_headers': "{'X-OpenAI-Api-Key': os.environ['OPENAI_API_KEY']}"}), "(url=WEAVIATE_URL, additional_headers={'X-OpenAI-Api-Key':\n    os.environ['OPENAI_API_KEY']})\n", (1009, 1102), False, 'import weaviate\n'), ((1464, 1504), 'langchain.vectorstores.Weaviate', 'Weaviate', (['client', '"""Paragraph"""', '"""content"""'], {}), "(client, 'Paragraph', 'content')\n", (1472, 1504), False, 'from langchain.vectorstores import Weaviate\n'), ((1827, 1856), 'os.getenv', 'os.getenv', (['"""AZURE_OPENAI_KEY"""'], {}), "('AZURE_OPENAI_KEY')\n", (1836, 1856), False, 'import os\n'), ((1874, 1908), 'os.getenv', 'os.getenv', (['"""AZURE_OPENAI_ENDPOINT"""'], {}), "('AZURE_OPENAI_ENDPOINT')\n", (1883, 1908), False, 'import os\n'), ((1925, 1951), 'os.getenv', 'os.getenv', (['"""RESOURCE_NAME"""'], {}), "('RESOURCE_NAME')\n", (1934, 1951), False, 'import os\n'), ((2862, 2891), 'os.listdir', 'os.listdir', (['subdirectory_path'], {}), '(subdirectory_path)\n', (2872, 2891), False, 'import os\n'), ((2016, 2050), 'os.getenv', 'os.getenv', (['"""AZURE_OPENAI_ENDPOINT"""'], {}), "('AZURE_OPENAI_ENDPOINT')\n", (2025, 2050), False, 'import os\n'), ((2064, 2093), 'os.getenv', 'os.getenv', (['"""AZURE_OPENAI_KEY"""'], {}), "('AZURE_OPENAI_KEY')\n", (2073, 2093), False, 'import os\n'), ((4802, 4881), 'weaviate.Client', 'weaviate.Client', ([], {'url': '"""http://localhost:8080"""', 'additional_headers': 'api_key_header'}), "(url='http://localhost:8080', additional_headers=api_key_header)\n", (4817, 4881), False, 'import weaviate\n'), ((2955, 2996), 'os.path.join', 'os.path.join', (['subdirectory_path', 'filename'], {}), '(subdirectory_path, filename)\n', (2967, 2996), False, 'import os\n'), ((7628, 7654), 'os.listdir', 'os.listdir', (['directory_path'], {}), '(directory_path)\n', (7638, 7654), False, 'import os\n'), ((10504, 10688), 'llama_index.llms.AzureOpenAI', 'llama_index.llms.AzureOpenAI', ([], {'model': '"""gpt-35-turbo-16k"""', 'deployment_name': '"""gpt-35-16k"""', 'api_key': 'azure_openai_key', 'azure_endpoint': 'azure_endpoint', 'api_version': '"""2023-03-15-preview"""'}), "(model='gpt-35-turbo-16k', deployment_name=\n    'gpt-35-16k', api_key=azure_openai_key, azure_endpoint=azure_endpoint,\n    api_version='2023-03-15-preview')\n", (10532, 10688), False, 'import llama_index\n'), ((10800, 11005), 'llama_index.embeddings.AzureOpenAIEmbedding', 'llama_index.embeddings.AzureOpenAIEmbedding', ([], {'model': '"""text-embedding-ada-002"""', 'deployment_name': '"""te-ada-002"""', 'api_key': 'azure_openai_key', 'azure_endpoint': 'azure_endpoint', 'api_version': '"""2023-03-15-preview"""'}), "(model='text-embedding-ada-002',\n    deployment_name='te-ada-002', api_key=azure_openai_key, azure_endpoint=\n    azure_endpoint, api_version='2023-03-15-preview')\n", (10843, 11005), False, 'import llama_index\n'), ((11121, 11183), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm': 'llm', 'embed_model': 'embed_model'}), '(llm=llm, embed_model=embed_model)\n', (11149, 11183), False, 'from llama_index import VectorStoreIndex, ServiceContext, set_global_service_context\n'), ((11213, 11308), 'llama_index.vector_stores.WeaviateVectorStore', 'WeaviateVectorStore', ([], {'weaviate_client': 'self.client', 'index_name': '"""Lectures"""', 'text_key': '"""content"""'}), "(weaviate_client=self.client, index_name='Lectures',\n    text_key='content')\n", (11232, 11308), False, 'from llama_index.vector_stores import WeaviateVectorStore\n'), ((11574, 11602), 'llama_index.response.pprint_utils.pprint_source_node', 'pprint_source_node', (['nodes[0]'], {}), '(nodes[0])\n', (11592, 11602), False, 'from llama_index.response.pprint_utils import pprint_source_node\n'), ((13511, 13541), 'json.dumps', 'json.dumps', (['response'], {'indent': '(2)'}), '(response, indent=2)\n', (13521, 13541), False, 'import json\n'), ((3067, 3087), 'fitz.open', 'fitz.open', (['file_path'], {}), '(file_path)\n', (3076, 3087), False, 'import fitz\n'), ((7692, 7734), 'os.path.join', 'os.path.join', (['directory_path', 'subdirectory'], {}), '(directory_path, subdirectory)\n', (7704, 7734), False, 'import os\n'), ((7754, 7786), 'os.path.isdir', 'os.path.isdir', (['subdirectory_path'], {}), '(subdirectory_path)\n', (7767, 7786), False, 'import os\n'), ((3234, 3287), 'unstructured.cleaners.core.clean', 'clean', (['page_text'], {'bullets': '(True)', 'extra_whitespace': '(True)'}), '(page_text, bullets=True, extra_whitespace=True)\n', (3239, 3287), False, 'from unstructured.cleaners.core import clean\n'), ((11359, 11445), 'llama_index.VectorStoreIndex.from_vector_store', 'VectorStoreIndex.from_vector_store', (['vector_store'], {'service_context': 'service_context'}), '(vector_store, service_context=\n    service_context)\n', (11393, 11445), False, 'from llama_index import VectorStoreIndex, ServiceContext, set_global_service_context\n'), ((3970, 4023), 'unstructured.cleaners.core.clean', 'clean', (['last_page'], {'bullets': '(True)', 'extra_whitespace': '(True)'}), '(last_page, bullets=True, extra_whitespace=True)\n', (3975, 4023), False, 'from unstructured.cleaners.core import clean\n'), ((9510, 9528), 'time.sleep', 'time.sleep', (['(2 ** j)'], {}), '(2 ** j)\n', (9520, 9528), False, 'import time\n')]