code: #
  # Graph Database Ingester For 5e SRD Content
  #
  import dotenv
  dotenv.load_dotenv()
  
  import json
  from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage
  from llama_index.core.extractors import TitleExtractor, QuestionsAnsweredExtractor
  from llama_index.core.query_engine.graph_query_engine import ComposableGraphQueryEngine
  from llama_index.extractors.entity import EntityExtractor
  from llama_index.core.ingestion import IngestionPipeline
  from llama_index.core.node_parser import TokenTextSplitter, MarkdownNodeParser
  from llama_index.core.storage.docstore import SimpleDocumentStore
  from os import path
  from src.documents import loadWebPages, parseWebPage, getLinks
  from src.db import graphStore, vectorStore, documentStore
  from src.file_utils import createOutputFile
  from src.llm import documentTitle, answerFinder, get_service_context
  from src.log_utils import debug
  from src.rebel import extract_triplets
  from typing import Sequence
  from datetime import datetime
  from markdownify import markdownify as md
  
  # 1. Download Site
  
  failedLinks = []
  
  TARGET = "https://www.5esrd.com"
  STORAGE = "./srd-store"
  
  def loadIndex():
    context = StorageContext.from_defaults(graph_store=graphStore, vector_store=vectorStore, docstore=documentStore)
    graph = load_graph_from_storage(context, 'root')
    
    return context,graph
  
  
  # context = loadIndex()
  # load(TARGET, context)
  
  # exit()
  
  title = " Web Site Document Loader "
  print("=" * len(title))
  print(title)
  print("=" * len(title))
  
  print("> Creating Output File")
  file = createOutputFile('./kg-output', 'srd-graph-result')
  
  def loadWebsitesIntoGraph(url: str, context: StorageContext, links: set = set()):
    file.write(f'\n### URL: {url}\n')
    
    debug("==> Loading Web URL [url: {}]".format(url))
    document = loadWebPages([url]).pop()
  
    documentDetails: dict[str, any] = {
      "id": document.doc_id,
      "text": document.text[:20]
    }
    file.write('LlamaIndex Document Details:\n')
    file.write(f'```json\n')
    file.write(f'{json.dumps(documentDetails, indent=2)}')
    file.write(f'\n```\n\n')
  
    debug("====> Parsing HTML [url: {}]".format(url))
    htmlDoc = parseWebPage(document.text);
  
    file.write('HTML Document:\n')
    file.write(f'```html\n')
    file.write(f'{htmlDoc}')
    file.write(f'\n```\n')
  
    debug("====> Creating Document [url: {}]".format(url))
    now = datetime.now()
    createDate = f'{now:%Y%m%d%H%M%S}'
    # markdown = md(str(htmlDoc), strip=['script']).replace('\n\n', '\n')
  
    parsedDocument = Document(
        text=htmlDoc.text,
        id_=url,
        metadata={ "source": url, "createdAt": createDate })
    
    nodes = metadataExtractor([parsedDocument])
  
    file.write('### Document Nodes w/ Metadata:\n')
    file.write(f'{parsedDocument.text}\n')
  
    context.docstore.add_documents(nodes, store_text=False)
  
    debug("====> Loading Into Graph DB [url: {}]".format(url))
  
    KnowledgeGraphIndex.build_index_from_nodes()
    KnowledgeGraphIndex.from_documents(
      documents=[parsedDocument],
      service_context=get_service_context(),
      kg_triplet_extract_fn=extract_triplets,
      storage_context=context,
      show_progress=True
    )
  
    debug("====> Getting Links [url: {}]".format(url))
    webPageLinks = getLinks(htmlDoc.encode_contents(formatter="html"), url=url)
  
    debug(f'====> Links found: {len(webPageLinks)}')
  
    exit()
  
    for link in webPageLinks:
      try:
        if (link not in links):
          links.add(link)
          loadWebsitesIntoGraph(link, context, links=links)
        else:
          debug("===> Link Already Loaded [url: {}]...Skipping".format(url))
      except Exception as error:
        print(f'{error}')
        failedLinks.append(link)
  
  def getDocumentStore(documents: Sequence[Document]):
    dirExists = path.isdir(STORAGE)
  
    if (dirExists):
      print("==> Existing Index Found, Updating...")
      context = StorageContext.from_defaults(persist_dir=STORAGE)
      index = load_index_from_storage(context, service_context=get_service_context())
  
      index.refresh(documents=documents)
      return context, index
    else:
      print("==> No Index Found, Creating...")
      # TODO => Switch to external document storage solution, possibly graph
      docStore = SimpleDocumentStore()
      docStore.add_documents(documents)
      storageContext = StorageContext.from_defaults(docstore=docStore)
      
      storageContext.persist(STORAGE)
  
      index = VectorStoreIndex.from_documents(
        documents,
        transformations=metadataExtractor(),
        storage_context=storageContext,
        service_context=get_service_context(),
        show_progress=True
      )
  
      storageContext.persist(STORAGE)
      return storageContext, index
  
  def getWebDocumentAndLinks(url: str, documents: list[Document] = [], links: set = set()):
    # This suggests a new url search, add the original link provided
    if (len(links) == 0):
      links.add(url)
    
    try:
      debug("==> Loading Web URL [url: {}]".format(url))
      htmlDoc = loadWebPages([url])
      debug("====> Parsing HTML [url: {}]".format(url))
      webPage = parseWebPage(htmlDoc[0].text)
      debug("====> Creating Document [url: {}]".format(url))
      documents.append(Document(text=webPage, doc_id=url))
      debug("====> Getting Links [url: {}]".format(url))
      webPageLinks = getLinks(htmlDoc[0].text, url=url)
  
      for link in webPageLinks:
        if (link not in links):
          links.add(link)
          getWebDocumentAndLinks(link, documents=documents, links=links)
        else:
          debug("===> Link Already Loaded [url: {}]...Skipping".format(url))
    except:
      failedLinks.append(url)
    finally:
      return documents, links
  
  def metadataExtractor(documents: list[Document]):
    # splitter = TokenTextSplitter(
    #   separator=" ", chunk_size=512, chunk_overlap=128
    # )
  
    splitter = MarkdownNodeParser()
  
    titleExtractor = TitleExtractor(nodes=5, llm=documentTitle)
    qaExtractor = QuestionsAnsweredExtractor(questions=3, llm=answerFinder)
    entity = EntityExtractor(prediction_threshold=0.75, label_entities=True)
  
    return IngestionPipeline(transformations=[splitter, titleExtractor, qaExtractor, entity]).run(documents=documents)
  
  print("> Loading Index from Neo4j")
  context, graph = loadIndex()
  
  print("> Loading Web Pages")
  file.write(f'**Target:** {TARGET}\n\n')
  
  file.write("""
  ---
  ## Loading Documents
  **Details on the documents loaded into the system**
  
  """)
  loadWebsitesIntoGraph(TARGET, context=context)
  # queryEngine = ComposableGraphQueryEngine(graph=graph)
  
  
  file.write("## Web Page Loading Result\n\n")
  file.write("| Item | Count |\n")
  file.write("| :-: | :-: |\n")
  file.write("| Documents | {} |\n".format(len(context.docstore.docs)))
  file.write("| Failed Downloads | {} |\n\n".format(len(failedLinks)))
  file.write("**Failed Links**\n".format(len(failedLinks)))
  file.write("```json\n[\n".format(len(failedLinks)))
  for badLink in failedLinks:
    file.write(" \"{}\",".format(badLink))
  file.write("]\n```\n\n---\n".format(len(failedLinks)))
  
  exit()
  
  # 2. Create Graph Index & Store
  print("> Process Documents")
  storageContext, index = getDocumentStore(docs)
  
  # 3. Make available
  queryEngine = index.as_query_engine(
    verbose=True,
    response_mode="tree_summarize"
  )
  
  print("> Querying Data")
  file.write("## Query Data\n")
  questionA = "What saving throws is a Fighter proficient using?"
  questionB = "How many attack dice is a greatsword?"
  questionC = "Which spell does the most lighting damage regardless of class?"
  
  file.write("Each question and it's evaluated answer\n\n")
  print("==> Question A")
  responseA = queryEngine.query(questionA)
  file.write("```\nQuestion: {}\n\nAnswer:\n{}\n```\n\n".format(questionA, responseA))
  
  print("==> Question B")
  responseB = queryEngine.query(questionB)
  file.write("```\nQuestion: {}\n\nAnswer:\n{}\n```\n\n".format(questionB, responseB))
  
  print("==> Question C")
  responseC = queryEngine.query(questionC)
  file.write("```\nQuestion: {}\n\nAnswer:\n{}\n```\n\n".format(questionC, responseC))
  
  print(" == Complete == ")
  # 4. Create Flask Server
apis: llama_index.core.StorageContext.from_defaults llama_index.core.extractors.TitleExtractor llama_index.core.storage.docstore.SimpleDocumentStore llama_index.core.ingestion.IngestionPipeline llama_index.core.node_parser.MarkdownNodeParser llama_index.core.load_graph_from_storage llama_index.extractors.entity.EntityExtractor llama_index.core.KnowledgeGraphIndex.build_index_from_nodes llama_index.core.extractors.QuestionsAnsweredExtractor llama_index.core.Document
extract_api: [((63, 83), 'dotenv.load_dotenv', 'dotenv.load_dotenv', ([], {}), '()\n', (81, 83), False, 'import dotenv\n'), ((1601, 1652), 'src.file_utils.createOutputFile', 'createOutputFile', (['"""./kg-output"""', '"""srd-graph-result"""'], {}), "('./kg-output', 'srd-graph-result')\n", (1617, 1652), False, 'from src.file_utils import createOutputFile\n'), ((1222, 1329), 'llama_index.core.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'graph_store': 'graphStore', 'vector_store': 'vectorStore', 'docstore': 'documentStore'}), '(graph_store=graphStore, vector_store=\n    vectorStore, docstore=documentStore)\n', (1250, 1329), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((1335, 1375), 'llama_index.core.load_graph_from_storage', 'load_graph_from_storage', (['context', '"""root"""'], {}), "(context, 'root')\n", (1358, 1375), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((2191, 2218), 'src.documents.parseWebPage', 'parseWebPage', (['document.text'], {}), '(document.text)\n', (2203, 2218), False, 'from src.documents import loadWebPages, parseWebPage, getLinks\n'), ((2399, 2413), 'datetime.datetime.now', 'datetime.now', ([], {}), '()\n', (2411, 2413), False, 'from datetime import datetime\n'), ((2543, 2634), 'llama_index.core.Document', 'Document', ([], {'text': 'htmlDoc.text', 'id_': 'url', 'metadata': "{'source': url, 'createdAt': createDate}"}), "(text=htmlDoc.text, id_=url, metadata={'source': url, 'createdAt':\n    createDate})\n", (2551, 2634), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((2917, 2961), 'llama_index.core.KnowledgeGraphIndex.build_index_from_nodes', 'KnowledgeGraphIndex.build_index_from_nodes', ([], {}), '()\n', (2959, 2961), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((3760, 3779), 'os.path.isdir', 'path.isdir', (['STORAGE'], {}), '(STORAGE)\n', (3770, 3779), False, 'from os import path\n'), ((5770, 5790), 'llama_index.core.node_parser.MarkdownNodeParser', 'MarkdownNodeParser', ([], {}), '()\n', (5788, 5790), False, 'from llama_index.core.node_parser import TokenTextSplitter, MarkdownNodeParser\n'), ((5811, 5853), 'llama_index.core.extractors.TitleExtractor', 'TitleExtractor', ([], {'nodes': '(5)', 'llm': 'documentTitle'}), '(nodes=5, llm=documentTitle)\n', (5825, 5853), False, 'from llama_index.core.extractors import TitleExtractor, QuestionsAnsweredExtractor\n'), ((5870, 5927), 'llama_index.core.extractors.QuestionsAnsweredExtractor', 'QuestionsAnsweredExtractor', ([], {'questions': '(3)', 'llm': 'answerFinder'}), '(questions=3, llm=answerFinder)\n', (5896, 5927), False, 'from llama_index.core.extractors import TitleExtractor, QuestionsAnsweredExtractor\n'), ((5939, 6002), 'llama_index.extractors.entity.EntityExtractor', 'EntityExtractor', ([], {'prediction_threshold': '(0.75)', 'label_entities': '(True)'}), '(prediction_threshold=0.75, label_entities=True)\n', (5954, 6002), False, 'from llama_index.extractors.entity import EntityExtractor\n'), ((3864, 3913), 'llama_index.core.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': 'STORAGE'}), '(persist_dir=STORAGE)\n', (3892, 3913), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((4207, 4228), 'llama_index.core.storage.docstore.SimpleDocumentStore', 'SimpleDocumentStore', ([], {}), '()\n', (4226, 4228), False, 'from llama_index.core.storage.docstore import SimpleDocumentStore\n'), ((4288, 4335), 'llama_index.core.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'docstore': 'docStore'}), '(docstore=docStore)\n', (4316, 4335), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((4947, 4966), 'src.documents.loadWebPages', 'loadWebPages', (['[url]'], {}), '([url])\n', (4959, 4966), False, 'from src.documents import loadWebPages, parseWebPage, getLinks\n'), ((5035, 5064), 'src.documents.parseWebPage', 'parseWebPage', (['htmlDoc[0].text'], {}), '(htmlDoc[0].text)\n', (5047, 5064), False, 'from src.documents import loadWebPages, parseWebPage, getLinks\n'), ((5255, 5289), 'src.documents.getLinks', 'getLinks', (['htmlDoc[0].text'], {'url': 'url'}), '(htmlDoc[0].text, url=url)\n', (5263, 5289), False, 'from src.documents import loadWebPages, parseWebPage, getLinks\n'), ((1841, 1860), 'src.documents.loadWebPages', 'loadWebPages', (['[url]'], {}), '([url])\n', (1853, 1860), False, 'from src.documents import loadWebPages, parseWebPage, getLinks\n'), ((3052, 3073), 'src.llm.get_service_context', 'get_service_context', ([], {}), '()\n', (3071, 3073), False, 'from src.llm import documentTitle, answerFinder, get_service_context\n'), ((5145, 5179), 'llama_index.core.Document', 'Document', ([], {'text': 'webPage', 'doc_id': 'url'}), '(text=webPage, doc_id=url)\n', (5153, 5179), False, 'from llama_index.core import Document, KnowledgeGraphIndex, StorageContext, VectorStoreIndex, load_index_from_storage, load_graph_from_storage\n'), ((6013, 6099), 'llama_index.core.ingestion.IngestionPipeline', 'IngestionPipeline', ([], {'transformations': '[splitter, titleExtractor, qaExtractor, entity]'}), '(transformations=[splitter, titleExtractor, qaExtractor,\n    entity])\n', (6030, 6099), False, 'from llama_index.core.ingestion import IngestionPipeline\n'), ((2058, 2095), 'json.dumps', 'json.dumps', (['documentDetails'], {'indent': '(2)'}), '(documentDetails, indent=2)\n', (2068, 2095), False, 'import json\n'), ((3975, 3996), 'src.llm.get_service_context', 'get_service_context', ([], {}), '()\n', (3994, 3996), False, 'from src.llm import documentTitle, answerFinder, get_service_context\n'), ((4543, 4564), 'src.llm.get_service_context', 'get_service_context', ([], {}), '()\n', (4562, 4564), False, 'from src.llm import documentTitle, answerFinder, get_service_context\n')]