code: import logging
  from llama_index.readers.base import BaseReader
  from llama_index.readers.schema.base import Document
  import requests
  from typing import List
  import re
  import os
  import logging
  from llama_index.readers.base import BaseReader
  from llama_index.readers.schema.base import Document
  import requests
  from typing import List
  import os
  import pandas as pd
  import openai
  import ast
  
  TWITTER_USERNAME = "shauryr"
  
  def generate_search_queries_prompt(question):
      """Generates the search queries prompt for the given question.
      Args: question (str): The question to generate the search queries prompt for
      Returns: str: The search queries prompt for the given question
      """
  
      return (
          f'Please generate four related search queries that align with the initial query: "{question}"'
          f'Each variation should be presented as a list of strings, following this format: ["query 1", "query 2", "query 3", "query 4"]'
      )
  
  
  def get_related_questions(query):
      research_template = """You are a search engine expert"""
              
      messages = [{
                  "role": "system",
                  "content": research_template
              }, {
                  "role": "user",
                  "content": generate_search_queries_prompt(query),
              }]
  
      response = openai.ChatCompletion.create(
      model="gpt-3.5-turbo",
      messages=messages,
      temperature=0.5,
      max_tokens=256
      )
      related_questions = get_questions(response.choices[0].message.content)
      related_questions.append(query)
      return related_questions
  
  def get_questions(response_text):
      data = response_text.split("\n")
      data = [ast.literal_eval(item)[0] for item in data]
      return data
  
  def get_unique_docs(docs):
      unique_docs_id = []
      unique_docs = []
      for doc in docs:
          if doc.extra_info['paperId'] not in unique_docs:
              unique_docs_id.append(doc.extra_info['paperId'])
              unique_docs.append(doc)
      return unique_docs
  
  class SemanticScholarReader(BaseReader):
      """
      A class to read and process data from Semantic Scholar API
      ...
  
      Methods
      -------
      __init__():
         Instantiate the SemanticScholar object
  
      load_data(query: str, limit: int = 10, returned_fields: list = ["title", "abstract", "venue", "year", "paperId", "citationCount", "openAccessPdf", "authors"]) -> list:
          Loads data from Semantic Scholar based on the query and returned_fields
  
      """
  
      def __init__(self, timeout=10, api_key=None, base_dir="pdfs"):
          """
          Instantiate the SemanticScholar object
          """
          from semanticscholar import SemanticScholar
          import arxiv
  
          self.arxiv = arxiv
          self.base_dir = base_dir
          self.s2 = SemanticScholar(timeout=timeout)
          # check for base dir
          if not os.path.exists(self.base_dir):
              os.makedirs(self.base_dir)
  
      def _clear_cache(self):
          """
          delete the .citation* folder
          """
          import shutil
  
          shutil.rmtree("./.citation*")
  
      def _download_pdf(self, paper_id, url: str, base_dir="pdfs"):
          logger = logging.getLogger()
          headers = {
              "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
          }
          # Making a GET request
          response = requests.get(url, headers=headers, stream=True)
          content_type = response.headers["Content-Type"]
  
          # As long as the content-type is application/pdf, this will download the file
          if "application/pdf" in content_type:
              os.makedirs(base_dir, exist_ok=True)
              file_path = os.path.join(base_dir, f"{paper_id}.pdf")
              # check if the file already exists
              if os.path.exists(file_path):
                  logger.info(f"{file_path} already exists")
                  return file_path
              with open(file_path, "wb") as file:
                  for chunk in response.iter_content(chunk_size=1024):
                      if chunk:
                          file.write(chunk)
              logger.info(f"Downloaded pdf from {url}")
              return file_path
          else:
              logger.warning(f"{url} was not downloaded: protected")
              return None
  
      def _get_full_text_docs(self, documents: List[Document]) -> List[Document]:
          from PyPDF2 import PdfReader
  
          """
          Gets the full text of the documents from Semantic Scholar
  
          Parameters
          ----------
          documents: list
              The list of Document object that contains the search results
  
          Returns
          -------
          list
              The list of Document object that contains the search results with full text
  
          Raises
          ------
          Exception
              If there is an error while getting the full text
  
          """
          full_text_docs = []
          for paper in documents:
              metadata = paper.extra_info
              url = metadata["openAccessPdf"]
              externalIds = metadata["externalIds"]
              paper_id = metadata["paperId"]
              file_path = None
              persist_dir = os.path.join(self.base_dir, f"{paper_id}.pdf")
              if url and not os.path.exists(persist_dir):
                  # Download the document first
                  file_path = self._download_pdf(metadata["paperId"], url, persist_dir)
  
              if (
                  not url
                  and externalIds
                  and "ArXiv" in externalIds
                  and not os.path.exists(persist_dir)
              ):
                  # download the pdf from arxiv
                  file_path = self._download_pdf_from_arxiv(
                      paper_id, externalIds["ArXiv"]
                  )
  
              # Then, check if it's a valid PDF. If it's not, skip to the next document.
              if file_path:
                  try:
                      pdf = PdfReader(open(file_path, "rb"))
                  except Exception as e:
                      logging.error(
                          f"Failed to read pdf with exception: {e}. Skipping document..."
                      )
                      continue
  
                  text = ""
                  for page in pdf.pages:
                      text += page.extract_text()
                  full_text_docs.append(Document(text=text, extra_info=metadata))
  
          return full_text_docs
  
      def _download_pdf_from_arxiv(self, paper_id, arxiv_id):
          paper = next(self.arxiv.Search(id_list=[arxiv_id], max_results=1).results())
          paper.download_pdf(dirpath=self.base_dir, filename=paper_id + ".pdf")
          return os.path.join(self.base_dir, f"{paper_id}.pdf")
  
      def load_data(
          self,
          query,
          limit,
          full_text=False,
          returned_fields=[
              "title",
              "abstract",
              "venue",
              "year",
              "paperId",
              "citationCount",
              "openAccessPdf",
              "authors",
              "externalIds",
          ],
      ) -> List[Document]:
          """
          Loads data from Semantic Scholar based on the entered query and returned_fields
  
          Parameters
          ----------
          query: str
              The search query for the paper
          limit: int, optional
              The number of maximum results returned (default is 10)
          returned_fields: list, optional
              The list of fields to be returned from the search
  
          Returns
          -------
          list
              The list of Document object that contains the search results
  
          Raises
          ------
          Exception
              If there is an error while performing the search
  
          """
          results = []
          # query = get_related_questions(query)
          query = [query]
          try:
              for question in query:
                  logging.info(f"Searching for {question}")
                  _results = self.s2.search_paper(question, limit=limit, fields=returned_fields)
                  results.extend(_results[:limit])
          except (requests.HTTPError, requests.ConnectionError, requests.Timeout) as e:
              logging.error(
                  "Failed to fetch data from Semantic Scholar with exception: %s", e
              )
              raise
          except Exception as e:
              logging.error("An unexpected error occurred: %s", e)
              raise
  
          documents = []
          
          for item in results[:limit*len(query)]:
              openAccessPdf = getattr(item, "openAccessPdf", None)
              abstract = getattr(item, "abstract", None)
              title = getattr(item, "title", None)
              text = None
              # concat title and abstract
              if abstract and title:
                  text = title + " " + abstract
              elif not abstract:
                  text = title
  
              metadata = {
                  "title": title,
                  "venue": getattr(item, "venue", None),
                  "year": getattr(item, "year", None),
                  "paperId": getattr(item, "paperId", None),
                  "citationCount": getattr(item, "citationCount", None),
                  "openAccessPdf": openAccessPdf.get("url") if openAccessPdf else None,
                  "authors": [author["name"] for author in getattr(item, "authors", [])],
                  "externalIds": getattr(item, "externalIds", None),
              }
              documents.append(Document(text=text, extra_info=metadata))
  
          if full_text:
              logging.info("Getting full text documents...")
              full_text_documents = self._get_full_text_docs(documents)
              documents.extend(full_text_documents)
          
          documents = get_unique_docs(documents)
          
          return documents
  
  
  def get_twitter_badge():
      """Constructs the Markdown code for the Twitter badge."""
      return f'<a href="https://twitter.com/{TWITTER_USERNAME}" target="_blank"><img src="https://img.shields.io/badge/Twitter-1DA1F2?style=for-the-badge&logo=twitter&logoColor=white" /></a>'
  
  
  def get_link_tree_badge():
      return f'<a href="https://linktr.ee/shauryr" target="_blank"><img src="https://img.shields.io/badge/Linktree-39E09B?style=for-the-badge&logo=linktree&logoColor=white" /></a>'
  
  def get_github_badge():
      return f'<a href="https://github.com/shauryr/s2qa" target="_blank"><img src="https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white" /></a>'
  
  def display_questions(sample_questions):
      s = "#### 🧐 More questions? \n"
      for i in sample_questions:
          s += "- " + i + "\n"
  
      return s
  
  
  def get_citation(metadata):
      # Extract details from metadata
      title = metadata.get("title", "No Title")
      venue = metadata.get("venue", "No Venue")
      year = metadata.get("year", "No Year")
      authors = metadata.get("authors", [])
  
      # Generate author names in correct format
      author_names = []
      for author in authors[:5]:
          last_name, *first_names = author.split(" ")
          first_initials = " ".join(name[0] + "." for name in first_names)
          author_names.append(f"{last_name}, {first_initials}")
  
      authors_string = ", & ".join(author_names)
  
      # APA citation format: Author1, Author2, & Author3. (Year). Title. Venue.
      citation = f"{authors_string}. ({year}). **{title}**. {venue}."
  
      return citation
  
  
  def extract_numbers_in_brackets(input_string):
      # use regular expressions to find all occurrences of [number]
      # numbers_in_brackets = re.findall(r"\[(\d+)\]", input_string)
      numbers_in_brackets = re.findall(r"\[(.*?)\]", input_string)
      # numbers_in_brackets = [int(i) for num in numbers_in_brackets for i in num.split(",")]
      # convert all numbers to int and remove duplicates by converting list to set and then back to list
      cleaned_numbers = []
      for n in numbers_in_brackets:
      # Try to convert the value to an integer
          try:
              cleaned_numbers.append(int(n))
          # If it fails (throws a ValueError), just ignore and continue with the next value
          except ValueError:
              continue
  
      # Apply the rest of your code on the cleaned list
      return sorted(list(set(cleaned_numbers)))
  
  
  def generate_used_reference_display(source_nodes, used_nodes):
      reference_display = "\n #### 📚 References: \n"
      # for index in used_nodes get the source node and add it to the reference display
      for index in used_nodes:
          try:
              source_node = source_nodes[index - 1]
          except IndexError:
              return "\n #### 😞 Couldnt Parse References \n"
          metadata = source_node.node.metadata
          reference_display += (
              "[["
              + str(source_nodes.index(source_node) + 1)
              + "]"
              + "("
              + "https://www.semanticscholar.org/paper/"
              + metadata["paperId"]
              + ")] "
              + "\n `. . ."
              + str(source_node.node.text)[100:290]
              + ". . .`"
              + get_citation(metadata)
              + " \n\n"
          )
  
      return reference_display
  
  def documents_to_df(documents):
      # convert document objects to dataframe
      list_data = []
      for i, doc in enumerate(documents):
          list_data.append(doc.extra_info.copy())
      
      df = pd.DataFrame(list_data)
      return df
          
  
  def generate_reference_display(source_nodes):
      reference_display = "\n ### References: \n"
      for source_node in source_nodes:
          metadata = source_node.node.metadata
          # add number infront of citation to make it easier to reference
          # reference_display += (
          #     "[["
          #     + str(source_nodes.index(source_node) + 1)
          #     + "]"
          #     + "("
          #     + "https://www.semanticscholar.org/paper/"
          #     + metadata["paperId"]
          #     + ")] "
          #     + '\n "`. . .'
          #     + str(source_node.node.text)[100:290]
          #     + ". . .` - **"
          #     + get_citation(metadata)
          #     + "** \n\n"
          # )
          reference_display += (
              "[["
              + str(source_nodes.index(source_node) + 1)
              + "]"
              + "("
              + "https://www.semanticscholar.org/paper/"
              + metadata["paperId"]
              + ")] "
              + get_citation(metadata)
              + " \n\n"
          )
      return reference_display
  
apis: llama_index.readers.schema.base.Document
extract_api: [((1299, 1406), 'openai.ChatCompletion.create', 'openai.ChatCompletion.create', ([], {'model': '"""gpt-3.5-turbo"""', 'messages': 'messages', 'temperature': '(0.5)', 'max_tokens': '(256)'}), "(model='gpt-3.5-turbo', messages=messages,\n    temperature=0.5, max_tokens=256)\n", (1327, 1406), False, 'import openai\n'), ((11530, 11569), 're.findall', 're.findall', (['"""\\\\[(.*?)\\\\]"""', 'input_string'], {}), "('\\\\[(.*?)\\\\]', input_string)\n", (11540, 11569), False, 'import re\n'), ((13232, 13255), 'pandas.DataFrame', 'pd.DataFrame', (['list_data'], {}), '(list_data)\n', (13244, 13255), True, 'import pandas as pd\n'), ((2741, 2773), 'semanticscholar.SemanticScholar', 'SemanticScholar', ([], {'timeout': 'timeout'}), '(timeout=timeout)\n', (2756, 2773), False, 'from semanticscholar import SemanticScholar\n'), ((3009, 3038), 'shutil.rmtree', 'shutil.rmtree', (['"""./.citation*"""'], {}), "('./.citation*')\n", (3022, 3038), False, 'import shutil\n'), ((3123, 3142), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (3140, 3142), False, 'import logging\n'), ((3366, 3413), 'requests.get', 'requests.get', (['url'], {'headers': 'headers', 'stream': '(True)'}), '(url, headers=headers, stream=True)\n', (3378, 3413), False, 'import requests\n'), ((6622, 6668), 'os.path.join', 'os.path.join', (['self.base_dir', 'f"""{paper_id}.pdf"""'], {}), "(self.base_dir, f'{paper_id}.pdf')\n", (6634, 6668), False, 'import os\n'), ((1649, 1671), 'ast.literal_eval', 'ast.literal_eval', (['item'], {}), '(item)\n', (1665, 1671), False, 'import ast\n'), ((2818, 2847), 'os.path.exists', 'os.path.exists', (['self.base_dir'], {}), '(self.base_dir)\n', (2832, 2847), False, 'import os\n'), ((2861, 2887), 'os.makedirs', 'os.makedirs', (['self.base_dir'], {}), '(self.base_dir)\n', (2872, 2887), False, 'import os\n'), ((3615, 3651), 'os.makedirs', 'os.makedirs', (['base_dir'], {'exist_ok': '(True)'}), '(base_dir, exist_ok=True)\n', (3626, 3651), False, 'import os\n'), ((3676, 3717), 'os.path.join', 'os.path.join', (['base_dir', 'f"""{paper_id}.pdf"""'], {}), "(base_dir, f'{paper_id}.pdf')\n", (3688, 3717), False, 'import os\n'), ((3780, 3805), 'os.path.exists', 'os.path.exists', (['file_path'], {}), '(file_path)\n', (3794, 3805), False, 'import os\n'), ((5158, 5204), 'os.path.join', 'os.path.join', (['self.base_dir', 'f"""{paper_id}.pdf"""'], {}), "(self.base_dir, f'{paper_id}.pdf')\n", (5170, 5204), False, 'import os\n'), ((9474, 9520), 'logging.info', 'logging.info', (['"""Getting full text documents..."""'], {}), "('Getting full text documents...')\n", (9486, 9520), False, 'import logging\n'), ((7834, 7875), 'logging.info', 'logging.info', (['f"""Searching for {question}"""'], {}), "(f'Searching for {question}')\n", (7846, 7875), False, 'import logging\n'), ((8118, 8203), 'logging.error', 'logging.error', (['"""Failed to fetch data from Semantic Scholar with exception: %s"""', 'e'], {}), "('Failed to fetch data from Semantic Scholar with exception: %s',\n    e)\n", (8131, 8203), False, 'import logging\n'), ((8291, 8343), 'logging.error', 'logging.error', (['"""An unexpected error occurred: %s"""', 'e'], {}), "('An unexpected error occurred: %s', e)\n", (8304, 8343), False, 'import logging\n'), ((9397, 9437), 'llama_index.readers.schema.base.Document', 'Document', ([], {'text': 'text', 'extra_info': 'metadata'}), '(text=text, extra_info=metadata)\n', (9405, 9437), False, 'from llama_index.readers.schema.base import Document\n'), ((5232, 5259), 'os.path.exists', 'os.path.exists', (['persist_dir'], {}), '(persist_dir)\n', (5246, 5259), False, 'import os\n'), ((5534, 5561), 'os.path.exists', 'os.path.exists', (['persist_dir'], {}), '(persist_dir)\n', (5548, 5561), False, 'import os\n'), ((6310, 6350), 'llama_index.readers.schema.base.Document', 'Document', ([], {'text': 'text', 'extra_info': 'metadata'}), '(text=text, extra_info=metadata)\n', (6318, 6350), False, 'from llama_index.readers.schema.base import Document\n'), ((6004, 6082), 'logging.error', 'logging.error', (['f"""Failed to read pdf with exception: {e}. Skipping document..."""'], {}), "(f'Failed to read pdf with exception: {e}. Skipping document...')\n", (6017, 6082), False, 'import logging\n')]