code: import streamlit as st
  import pandas as pd
  import json
  import requests
  from pathlib import Path
  from datetime import datetime
  from jinja2 import Template
  import lancedb
  import sqlite3
  from services.lancedb_notes import IndexDocumentsNotes
  
  st.set_page_config(layout='wide',
                     page_title='Notes')
  
  @st.cache_data
  def summarize_text(text, prompt):
      system_prompt = '\n'.join(prompt['system_prompt'])
      content_prompt = Template('\n'.join(prompt['content_prompt']))
      content_prompt = content_prompt.render({'text':text})
      results = requests.post('http://localhost:8001/summarize',
                              json={'system_prompt':system_prompt, 'content_prompt':content_prompt})
      return results.json()['summary']
  
  # @st.cache_data
  def keyphrase_text(text, strength_cutoff=0.3):
      results = requests.post('http://localhost:8001/phrases',
                              json={'text':text})
      results = results.json()
      phrases = list()
      for phrase, strength in zip(results['phrases'], results['strengths']):
          if strength >= strength_cutoff:
              phrases.append(phrase)
      return phrases
  
  @st.cache_data
  def entities_extract(text):
      results = requests.post('http://localhost:8001/ner',
                              json={'text':text})
      # st.write(results.json()['entities'])
      return results.json()['entities']
  
  @st.cache_data
  def store_note(note):
      with open('tmp_json.json','w') as f:
          json.dump(note, f)
      return note
  
  
  notes_folder = Path('data/notes')
  collections_folder = Path('data/collections')
  tmp_folder = Path('data/tmp')
  config_folder = Path('data/config')
  with open(config_folder.joinpath('prompt_templates.json'), 'r') as f:
      prompt_options = json.load(f)
  
  
  index_folder = Path('indexes')
  sqlite_location = Path('data/indexes/documents.sqlite')
  
  lance_index = lancedb.connect(index_folder)
  available_indexes = lance_index.table_names()
  selected_collections = st.multiselect('Which note collections to load', options=available_indexes)
  
  index_to_search = st.selectbox(label='Available Indexes', options=available_indexes)
  prompt_option_choices = [x['Name'] for x in prompt_options]
  
  
  for collection_idx, collection_name in enumerate(selected_collections):
      sqlite_conn = sqlite3.connect(sqlite_location)
      notes = sqlite_conn.execute(f"""SELECT * from {collection_name}""").fetchall()
      fields = sqlite_conn.execute(f"PRAGMA table_info({collection_name})").fetchall()
      fields = [x[1] for x in fields]
      notes = [dict(zip(fields, note)) for note in notes]
      for note in notes:
          note['metadata'] = json.loads(note['metadata'])
  
      with st.expander(collection_name):
          st.markdown("""Batch processing is possible. Select what you want and whether you want to overwrite the files or save to a new collection.\n\n*If you want to overwrite, leave the collection name as is.*""")
          batch_phrase, batch_entity, batch_summary = st.columns([1,1,1])
          with batch_phrase:
              batch_phrase_extract = st.toggle('Batch Phrase Extract', key=f'batch_phrase_extract_{collection_name}')
          with batch_entity:
              batch_entity_extract = st.toggle('Batch Entity Extract', key=f'batch_entity_extract_{collection_name}')
          with batch_summary:
              batch_summary_extract = st.toggle('Batch Summary Extract', key=f'batch_summary_extract_{collection_name}')
  
          selected_prompt_name = st.selectbox("Which prompt template?", prompt_option_choices, index=0)
          selected_prompt = prompt_options[prompt_option_choices.index(selected_prompt_name)]
          print(selected_prompt)
          save_collection_name = st.text_input('Saved Notes Collection Name', value=collection_name, key=f'batch_collection_save_{collection_name}')
  
          if st.button('Batch Process!', key=f'batch_process_{collection_name}'):
              progress_text = "Processing Progress (May take some time if summarizing)"
              batch_progress_bar = st.progress(0, text=progress_text)
              for i, note in enumerate(notes, start=1):
                  if batch_entity_extract:
                      entities = entities_extract(note['text'])
                      note['entities'] = entities
  
                  if batch_summary_extract:
                      note['summary'] = summarize_text(note['text'], selected_prompt).strip()
  
                  batch_progress_bar.progress(i/len(notes), text=progress_text)
              st.write("Collection Processed!")
  
          with st.container():
              for index, note in enumerate(notes):
                  st.markdown(f"**:blue[{note['title']}]**")
                  if st.toggle('Show Note', key=f'show_note_{collection_name}_{index}'):
                      text_col, note_col = st.columns([0.6,0.4])
                      with text_col:
                          st.markdown(f"**Date:** {note['date']}")
                          st.markdown(f"**Title:** {note['title']}")
                          if 'tags' in note and len(note['tags']) > 0:
                              st.markdown(f"**Tags:** {note['tags']}")
                          if 'phrases' in note['metadata']:
                              st.markdown(f"**Keyphrases:** {note['metadata']['phrases']}")
                          if 'entities' in note['metadata']:
                              st.markdown(f"Entities:** {note['metadata']['entities']}")
  
                          st.markdown("**Text**")
                          st.markdown(note['text'].replace('\n','\n\n'))
                          st.json(note['metadata'], expanded=False)
  
                      with note_col:
                          ## Create session state for the text
                          ## add button to make rest api call to populate and then update text
                          ## Add button to save the note
                          save_note, local_collection = st.columns([1,3])
  
                          with save_note:
                              _save_note = st.button('Save', key=f'save_note_{collection_name}_{index}')
  
                          ### Keyphrase extraction using Keybert/Keyphrase Vectorizers/Spacy NLP
                          if st.toggle('\nPhrase Extract', key=f'phrase_extract_{collection_name}_{index}'):
                              phrases = keyphrase_text(note['text'])
                              if 'phrases' not in note['metadata']:
                                  note['metadata']['phrases'] = ','.join(phrases)
                              else:
                                  note['metadata']['phrases'] = note['metadata']['phrases'] +'\n' + ','.join(phrases)
  
                          if 'phrases' in note['metadata']:
                              note['metadata']['phrases']  = st.text_area('Keyphrases', value=note['metadata']['phrases'],
                                                                  height=100, key=f'phrase_input_{collection_name}_{index}')
                          else:
                              note['metadata']['phrases']  = st.text_area('Keyphrases', value='',
                                                                  height=100, key=f'phrase_input_{collection_name}_{index}')
  
                          ### Entity extraction using Spacy NLP backend
                          if st.toggle('Entity Extract', key=f'entity_extract_{collection_name}_{index}'):
                              if 'entities' not in note['metadata']:
                                  note['metadata']['entities'] = dict()
                              entities = entities_extract(note['text'])
                              note['metadata']['entities'].update(entities)
                          # st.write(note['metadata']['entities'])
                          entities_formatted = ''
                          if 'entities' in note['metadata']:
                              entities_formatted = ''
                              for ent_type, ents in note['metadata']['entities'].items():
                                  ents_text = ', '.join(ents)
                                  entities_formatted += f'{ent_type}: {ents_text};\n\n'
                              entities_formatted = entities_formatted.strip()
                              entities_formatted = st.text_area('Entities', value=entities_formatted,
                                                                  height=200, key=f'entity_input_{collection_name}_{index}')
                          else:
                              entities = st.text_area('Entities', value='',
                                                                  height=200, key=f'entity_input_{collection_name}_{index}')
                          note_json = dict()
                          for entity in entities_formatted.split(';'):
                              if len(entity) == 0:
                                  continue
                              entity_type, entity_values = entity.split(':')
                              entity_values = [x.strip() for x in entity_values.split(',')]
                              note_json[entity_type.strip()] = entity_values
                          note['metadata']['entities'] = note_json
  
                          #### Summarization using Llama CPP backend
                          selected_prompt_name = st.selectbox("Which prompt template?", prompt_option_choices, index=0,
                                                              key=f'doc_prompt_template_{collection_name}_{index}')
                          selected_prompt = prompt_options[prompt_option_choices.index(selected_prompt_name)]
                          if st.toggle('Summarize', key=f'summary_extract_{collection_name}_{index}'):
                              if 'summary' not in note['metadata']:
                                  note['metadata']['summary'] = ''
                              summary = summarize_text(note['text'], selected_prompt).strip()
                              note['metadata']['summary'] = summary
                          if 'summary' in note['metadata']:
                              note['metadata']['summary'] = st.text_area('Summary', value=note['metadata']['summary'], height=500,
                                                                 key=f'summary_input_{collection_name}_{index}')
                          else:
                              note['metadata']['summary'] = st.text_area('Summary', value='', height=500,
                                                                 key=f'summary_input_{collection_name}_{index}')
  
                          if _save_note:
                              note['metadata'] = json.dumps(note['metadata'])
                              lance_table = lance_index.open_table(collection_name)
                              st.write(note['uuid'])
                              # LanceDB current can't (or more likely I don't know) how to update its metadata fields
                              # Sqlite will be used instead as it's the document repository anyways
                              # To create searchable notes, I'll have to think up something with lancedb_notes
                              # lance_table.update(where=f"uuid =' {note['uuid']}'", values={'metadata':note['metadata']})
                              sqlite_conn.execute(f"""UPDATE {collection_name} SET metadata='{note['metadata'].replace("'","''")}' WHERE uuid='{note['uuid']}'""")
                              sqlite_conn.commit()
  
  with st.sidebar:
      new_collection_name = st.text_input(label='New Collection Name', value='')
      if st.button('Create Collection'):
          collections_folder.joinpath(new_collection_name).mkdir(parents=True, exist_ok=True)
          st.rerun()
apis: lancedb.connect
extract_api: [((240, 293), 'streamlit.set_page_config', 'st.set_page_config', ([], {'layout': '"""wide"""', 'page_title': '"""Notes"""'}), "(layout='wide', page_title='Notes')\n", (258, 293), True, 'import streamlit as st\n'), ((1504, 1522), 'pathlib.Path', 'Path', (['"""data/notes"""'], {}), "('data/notes')\n", (1508, 1522), False, 'from pathlib import Path\n'), ((1544, 1568), 'pathlib.Path', 'Path', (['"""data/collections"""'], {}), "('data/collections')\n", (1548, 1568), False, 'from pathlib import Path\n'), ((1582, 1598), 'pathlib.Path', 'Path', (['"""data/tmp"""'], {}), "('data/tmp')\n", (1586, 1598), False, 'from pathlib import Path\n'), ((1615, 1634), 'pathlib.Path', 'Path', (['"""data/config"""'], {}), "('data/config')\n", (1619, 1634), False, 'from pathlib import Path\n'), ((1756, 1771), 'pathlib.Path', 'Path', (['"""indexes"""'], {}), "('indexes')\n", (1760, 1771), False, 'from pathlib import Path\n'), ((1790, 1827), 'pathlib.Path', 'Path', (['"""data/indexes/documents.sqlite"""'], {}), "('data/indexes/documents.sqlite')\n", (1794, 1827), False, 'from pathlib import Path\n'), ((1843, 1872), 'lancedb.connect', 'lancedb.connect', (['index_folder'], {}), '(index_folder)\n', (1858, 1872), False, 'import lancedb\n'), ((1942, 2017), 'streamlit.multiselect', 'st.multiselect', (['"""Which note collections to load"""'], {'options': 'available_indexes'}), "('Which note collections to load', options=available_indexes)\n", (1956, 2017), True, 'import streamlit as st\n'), ((2037, 2103), 'streamlit.selectbox', 'st.selectbox', ([], {'label': '"""Available Indexes"""', 'options': 'available_indexes'}), "(label='Available Indexes', options=available_indexes)\n", (2049, 2103), True, 'import streamlit as st\n'), ((557, 682), 'requests.post', 'requests.post', (['"""http://localhost:8001/summarize"""'], {'json': "{'system_prompt': system_prompt, 'content_prompt': content_prompt}"}), "('http://localhost:8001/summarize', json={'system_prompt':\n    system_prompt, 'content_prompt': content_prompt})\n", (570, 682), False, 'import requests\n'), ((821, 888), 'requests.post', 'requests.post', (['"""http://localhost:8001/phrases"""'], {'json': "{'text': text}"}), "('http://localhost:8001/phrases', json={'text': text})\n", (834, 888), False, 'import requests\n'), ((1193, 1256), 'requests.post', 'requests.post', (['"""http://localhost:8001/ner"""'], {'json': "{'text': text}"}), "('http://localhost:8001/ner', json={'text': text})\n", (1206, 1256), False, 'import requests\n'), ((1726, 1738), 'json.load', 'json.load', (['f'], {}), '(f)\n', (1735, 1738), False, 'import json\n'), ((2256, 2288), 'sqlite3.connect', 'sqlite3.connect', (['sqlite_location'], {}), '(sqlite_location)\n', (2271, 2288), False, 'import sqlite3\n'), ((11349, 11401), 'streamlit.text_input', 'st.text_input', ([], {'label': '"""New Collection Name"""', 'value': '""""""'}), "(label='New Collection Name', value='')\n", (11362, 11401), True, 'import streamlit as st\n'), ((11409, 11439), 'streamlit.button', 'st.button', (['"""Create Collection"""'], {}), "('Create Collection')\n", (11418, 11439), True, 'import streamlit as st\n'), ((1452, 1470), 'json.dump', 'json.dump', (['note', 'f'], {}), '(note, f)\n', (1461, 1470), False, 'import json\n'), ((2599, 2627), 'json.loads', 'json.loads', (["note['metadata']"], {}), "(note['metadata'])\n", (2609, 2627), False, 'import json\n'), ((2638, 2666), 'streamlit.expander', 'st.expander', (['collection_name'], {}), '(collection_name)\n', (2649, 2666), True, 'import streamlit as st\n'), ((2676, 2890), 'streamlit.markdown', 'st.markdown', (['"""Batch processing is possible. Select what you want and whether you want to overwrite the files or save to a new collection.\n\n*If you want to overwrite, leave the collection name as is.*"""'], {}), '(\n    """Batch processing is possible. Select what you want and whether you want to overwrite the files or save to a new collection.\n\n*If you want to overwrite, leave the collection name as is.*"""\n    )\n', (2687, 2890), True, 'import streamlit as st\n'), ((2935, 2956), 'streamlit.columns', 'st.columns', (['[1, 1, 1]'], {}), '([1, 1, 1])\n', (2945, 2956), True, 'import streamlit as st\n'), ((3420, 3490), 'streamlit.selectbox', 'st.selectbox', (['"""Which prompt template?"""', 'prompt_option_choices'], {'index': '(0)'}), "('Which prompt template?', prompt_option_choices, index=0)\n", (3432, 3490), True, 'import streamlit as st\n'), ((3645, 3765), 'streamlit.text_input', 'st.text_input', (['"""Saved Notes Collection Name"""'], {'value': 'collection_name', 'key': 'f"""batch_collection_save_{collection_name}"""'}), "('Saved Notes Collection Name', value=collection_name, key=\n    f'batch_collection_save_{collection_name}')\n", (3658, 3765), True, 'import streamlit as st\n'), ((3773, 3840), 'streamlit.button', 'st.button', (['"""Batch Process!"""'], {'key': 'f"""batch_process_{collection_name}"""'}), "('Batch Process!', key=f'batch_process_{collection_name}')\n", (3782, 3840), True, 'import streamlit as st\n'), ((11541, 11551), 'streamlit.rerun', 'st.rerun', ([], {}), '()\n', (11549, 11551), True, 'import streamlit as st\n'), ((3017, 3102), 'streamlit.toggle', 'st.toggle', (['"""Batch Phrase Extract"""'], {'key': 'f"""batch_phrase_extract_{collection_name}"""'}), "('Batch Phrase Extract', key=f'batch_phrase_extract_{collection_name}'\n    )\n", (3026, 3102), True, 'import streamlit as st\n'), ((3160, 3245), 'streamlit.toggle', 'st.toggle', (['"""Batch Entity Extract"""'], {'key': 'f"""batch_entity_extract_{collection_name}"""'}), "('Batch Entity Extract', key=f'batch_entity_extract_{collection_name}'\n    )\n", (3169, 3245), True, 'import streamlit as st\n'), ((3305, 3392), 'streamlit.toggle', 'st.toggle', (['"""Batch Summary Extract"""'], {'key': 'f"""batch_summary_extract_{collection_name}"""'}), "('Batch Summary Extract', key=\n    f'batch_summary_extract_{collection_name}')\n", (3314, 3392), True, 'import streamlit as st\n'), ((3961, 3995), 'streamlit.progress', 'st.progress', (['(0)'], {'text': 'progress_text'}), '(0, text=progress_text)\n', (3972, 3995), True, 'import streamlit as st\n'), ((4427, 4460), 'streamlit.write', 'st.write', (['"""Collection Processed!"""'], {}), "('Collection Processed!')\n", (4435, 4460), True, 'import streamlit as st\n'), ((4475, 4489), 'streamlit.container', 'st.container', ([], {}), '()\n', (4487, 4489), True, 'import streamlit as st\n'), ((4556, 4598), 'streamlit.markdown', 'st.markdown', (['f"""**:blue[{note[\'title\']}]**"""'], {}), '(f"**:blue[{note[\'title\']}]**")\n', (4567, 4598), True, 'import streamlit as st\n'), ((4618, 4684), 'streamlit.toggle', 'st.toggle', (['"""Show Note"""'], {'key': 'f"""show_note_{collection_name}_{index}"""'}), "('Show Note', key=f'show_note_{collection_name}_{index}')\n", (4627, 4684), True, 'import streamlit as st\n'), ((4727, 4749), 'streamlit.columns', 'st.columns', (['[0.6, 0.4]'], {}), '([0.6, 0.4])\n', (4737, 4749), True, 'import streamlit as st\n'), ((4808, 4848), 'streamlit.markdown', 'st.markdown', (['f"""**Date:** {note[\'date\']}"""'], {}), '(f"**Date:** {note[\'date\']}")\n', (4819, 4848), True, 'import streamlit as st\n'), ((4873, 4915), 'streamlit.markdown', 'st.markdown', (['f"""**Title:** {note[\'title\']}"""'], {}), '(f"**Title:** {note[\'title\']}")\n', (4884, 4915), True, 'import streamlit as st\n'), ((5373, 5396), 'streamlit.markdown', 'st.markdown', (['"""**Text**"""'], {}), "('**Text**')\n", (5384, 5396), True, 'import streamlit as st\n'), ((5492, 5533), 'streamlit.json', 'st.json', (["note['metadata']"], {'expanded': '(False)'}), "(note['metadata'], expanded=False)\n", (5499, 5533), True, 'import streamlit as st\n'), ((5833, 5851), 'streamlit.columns', 'st.columns', (['[1, 3]'], {}), '([1, 3])\n', (5843, 5851), True, 'import streamlit as st\n'), ((6118, 6196), 'streamlit.toggle', 'st.toggle', (['"""\nPhrase Extract"""'], {'key': 'f"""phrase_extract_{collection_name}_{index}"""'}), "('\\nPhrase Extract', key=f'phrase_extract_{collection_name}_{index}')\n", (6127, 6196), True, 'import streamlit as st\n'), ((7211, 7287), 'streamlit.toggle', 'st.toggle', (['"""Entity Extract"""'], {'key': 'f"""entity_extract_{collection_name}_{index}"""'}), "('Entity Extract', key=f'entity_extract_{collection_name}_{index}')\n", (7220, 7287), True, 'import streamlit as st\n'), ((9176, 9305), 'streamlit.selectbox', 'st.selectbox', (['"""Which prompt template?"""', 'prompt_option_choices'], {'index': '(0)', 'key': 'f"""doc_prompt_template_{collection_name}_{index}"""'}), "('Which prompt template?', prompt_option_choices, index=0, key=\n    f'doc_prompt_template_{collection_name}_{index}')\n", (9188, 9305), True, 'import streamlit as st\n'), ((9496, 9568), 'streamlit.toggle', 'st.toggle', (['"""Summarize"""'], {'key': 'f"""summary_extract_{collection_name}_{index}"""'}), "('Summarize', key=f'summary_extract_{collection_name}_{index}')\n", (9505, 9568), True, 'import streamlit as st\n'), ((5013, 5053), 'streamlit.markdown', 'st.markdown', (['f"""**Tags:** {note[\'tags\']}"""'], {}), '(f"**Tags:** {note[\'tags\']}")\n', (5024, 5053), True, 'import streamlit as st\n'), ((5140, 5201), 'streamlit.markdown', 'st.markdown', (['f"""**Keyphrases:** {note[\'metadata\'][\'phrases\']}"""'], {}), '(f"**Keyphrases:** {note[\'metadata\'][\'phrases\']}")\n', (5151, 5201), True, 'import streamlit as st\n'), ((5289, 5347), 'streamlit.markdown', 'st.markdown', (['f"""Entities:** {note[\'metadata\'][\'entities\']}"""'], {}), '(f"Entities:** {note[\'metadata\'][\'entities\']}")\n', (5300, 5347), True, 'import streamlit as st\n'), ((5933, 5994), 'streamlit.button', 'st.button', (['"""Save"""'], {'key': 'f"""save_note_{collection_name}_{index}"""'}), "('Save', key=f'save_note_{collection_name}_{index}')\n", (5942, 5994), True, 'import streamlit as st\n'), ((6679, 6803), 'streamlit.text_area', 'st.text_area', (['"""Keyphrases"""'], {'value': "note['metadata']['phrases']", 'height': '(100)', 'key': 'f"""phrase_input_{collection_name}_{index}"""'}), "('Keyphrases', value=note['metadata']['phrases'], height=100,\n    key=f'phrase_input_{collection_name}_{index}')\n", (6691, 6803), True, 'import streamlit as st\n'), ((6953, 7053), 'streamlit.text_area', 'st.text_area', (['"""Keyphrases"""'], {'value': '""""""', 'height': '(100)', 'key': 'f"""phrase_input_{collection_name}_{index}"""'}), "('Keyphrases', value='', height=100, key=\n    f'phrase_input_{collection_name}_{index}')\n", (6965, 7053), True, 'import streamlit as st\n'), ((8153, 8267), 'streamlit.text_area', 'st.text_area', (['"""Entities"""'], {'value': 'entities_formatted', 'height': '(200)', 'key': 'f"""entity_input_{collection_name}_{index}"""'}), "('Entities', value=entities_formatted, height=200, key=\n    f'entity_input_{collection_name}_{index}')\n", (8165, 8267), True, 'import streamlit as st\n'), ((8396, 8494), 'streamlit.text_area', 'st.text_area', (['"""Entities"""'], {'value': '""""""', 'height': '(200)', 'key': 'f"""entity_input_{collection_name}_{index}"""'}), "('Entities', value='', height=200, key=\n    f'entity_input_{collection_name}_{index}')\n", (8408, 8494), True, 'import streamlit as st\n'), ((9975, 10098), 'streamlit.text_area', 'st.text_area', (['"""Summary"""'], {'value': "note['metadata']['summary']", 'height': '(500)', 'key': 'f"""summary_input_{collection_name}_{index}"""'}), "('Summary', value=note['metadata']['summary'], height=500, key=\n    f'summary_input_{collection_name}_{index}')\n", (9987, 10098), True, 'import streamlit as st\n'), ((10245, 10343), 'streamlit.text_area', 'st.text_area', (['"""Summary"""'], {'value': '""""""', 'height': '(500)', 'key': 'f"""summary_input_{collection_name}_{index}"""'}), "('Summary', value='', height=500, key=\n    f'summary_input_{collection_name}_{index}')\n", (10257, 10343), True, 'import streamlit as st\n'), ((10489, 10517), 'json.dumps', 'json.dumps', (["note['metadata']"], {}), "(note['metadata'])\n", (10499, 10517), False, 'import json\n'), ((10628, 10650), 'streamlit.write', 'st.write', (["note['uuid']"], {}), "(note['uuid'])\n", (10636, 10650), True, 'import streamlit as st\n')]